{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "id": "dgBTU57OWYI0"
   },
   "source": [
    "# Install Library\n",
    "\n",
    "[RDKit ](https://github.com/rdkit/rdkit)\n",
    "\n",
    "[DGL](https://github.com/dmlc/dgl/)\n",
    "\n",
    "[DGL-LifeSci](https://github.com/awslabs/dgl-lifesci)\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true,
    "id": "EOF1QxeqhajG"
   },
   "outputs": [],
   "source": [
    "%%capture\n",
    "!pip install rdkit-pypi\n",
    "!pip install dgllife\n",
    "!pip install --pre dgl-cu113 dglgo -f https://data.dgl.ai/wheels-test/repo.html"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "id": "xtojkovzWYI2"
   },
   "source": [
    "# Import Library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import dgl \n",
    "import sys\n",
    "import random\n",
    "import torch\n",
    "import cv2\n",
    "import torchvision\n",
    "import statistics\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "from dgllife.model import MLPPredictor\n",
    "from tensorflow.keras.callbacks import  History\n",
    "from dgllife.utils import smiles_to_bigraph, CanonicalAtomFeaturizer, AttentiveFPAtomFeaturizer\n",
    "from tqdm.notebook import tqdm, trange\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "from utils.general import DATASET, get_dataset, separate_active_and_inactive_data, get_embedding_vector_class, count_lablel,data_generator, up_and_down_Samplenig\n",
    "from utils.gcn_pre_trained import get_tox21_model\n",
    "from model.heterogeneous_siamese_tox21 import siamese_model_attentiveFp_tox21, siamese_model_Canonical_tox21\n",
    "\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "id": "1RVgRpTmQ5rp",
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "# Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "cache_path='./tox21_dglgraph.bin'\n",
    "\n",
    "df = get_dataset(\"tox21\")\n",
    "id = df['mol_id']\n",
    "df = df.drop(columns=['mol_id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "hidden": true,
    "id": "m_nAHT_WhajK"
   },
   "outputs": [],
   "source": [
    "tox21_tasks = df.columns.values[:12].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "hidden": true,
    "id": "GEdp1FUphajL",
    "outputId": "352d95e6-5d5c-4c98-8d51-d7f330c51e9f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NR-AR one: 309  zero: 6956  NAN: 566\n",
      "NR-AR-LBD one: 237  zero: 6521  NAN: 1073\n",
      "NR-AhR one: 768  zero: 5781  NAN: 1282\n",
      "NR-Aromatase one: 300  zero: 5521  NAN: 2010\n",
      "NR-ER one: 793  zero: 5400  NAN: 1638\n",
      "NR-ER-LBD one: 350  zero: 6605  NAN: 876\n",
      "NR-PPAR-gamma one: 186  zero: 6264  NAN: 1381\n",
      "SR-ARE one: 942  zero: 4890  NAN: 1999\n",
      "SR-ATAD5 one: 264  zero: 6808  NAN: 759\n",
      "SR-HSE one: 372  zero: 6095  NAN: 1364\n",
      "SR-MMP one: 918  zero: 4892  NAN: 2021\n",
      "SR-p53 one: 423  zero: 6351  NAN: 1057\n"
     ]
    }
   ],
   "source": [
    "one = []\n",
    "zero = []\n",
    "nan = []\n",
    " \n",
    "for task in tox21_tasks:\n",
    "    a = list(df[task].value_counts(dropna=False).to_dict().values())\n",
    "    zero.append(a[0])\n",
    "    nan.append(a[1])\n",
    "    one.append(a[2])\n",
    "    print(task ,\"one:\" ,a[2] ,\" zero:\", a[0], \" NAN:\",a[1])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5862, 72084)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(one), sum(zero)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 881
    },
    "hidden": true,
    "id": "QjNMCFSqhajM",
    "outputId": "112c0f92-1e31-467c-e3b5-a9285f9c9f14"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABmIAAATWCAYAAAA2BLk9AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAA9hAAAPYQGoP6dpAAB/a0lEQVR4nOzde7TVdYH//9eGw51zjqLCESWhRPKat0J0JnFAJDWdrLyAiHlNK4dJ06xZA2lh2PJS4TimJHnLatSyG94yyxQvOJi3rClETVBUPCgiIO7fH37dv46gAvJmy+HxWGuv1fns9/583p/9bucan/Peu1KtVqsBAAAAAABgjetQ7wkAAAAAAAC0V0IMAAAAAABAIUIMAAAAAABAIUIMAAAAAABAIUIMAAAAAABAIUIMAAAAAABAIUIMAAAAAABAIUIMAAAAAABAIUIMAAAAAABAIUIMAACsgyqVyko9fvvb366R61177bU57LDDsuWWW6Zbt27p379/Ro8enb/85S/Ljf3FL36RI444Ittvv306deqUSqXyrq//2GOPpVKpZOrUqav82ocffjgTJkzIY4899q7nsSbccccdmTBhQl544YV3dZ4jjzwy/fv3b3OsUqlkwoQJ7+q8AADAmtVQ7wkAAACr7s4772zz95lnnplbb701v/nNb9oc32abbdbI9SZNmpSWlpZ89atfzfvf//488cQTmThxYnbeeedMnz492267bW3sddddl+nTp2ennXZKly5dMmPGjDUyh9X18MMP52tf+1qGDh26XLiohzvuuCNf+9rXcuSRR2aDDTZYo+e+8847s/nmm6/RcwIAAO+OEAMAAOug3Xbbrc3fm2yySTp06LDc8TXl5z//eXr37t3m2L/8y7+kf//+Oe+883LJJZfUjl988cXp0OH1zfef//zn6x5i1iel1v+dvPzyy+nevXtdrg0AAO91vpoMAADaqeeffz4nnnhiNttss3Tu3Dnvf//789WvfjWLFy9OkrzyyivZaaedsuWWW6a1tbX2urlz56alpSVDhw7NsmXLkmS5CJMkffv2zeabb54nnniizfE3Iszqeuqpp3LwwQensbExzc3NOeSQQzJ37tzlxt1777059NBD079//9rXpR122GGZPXt2bczUqVPz6U9/Okmy11571b6y7Y2vOLvpppty4IEHZvPNN0/Xrl2z5ZZb5vjjj8+zzz7b5lrz5s3Lcccdl379+qVLly7ZZJNNsscee+Tmm29uM+7mm2/OsGHD0tTUlO7du2ePPfbILbfcUnt+woQJ+dKXvpQkGTBgwEp/hdzUqVMzaNCgdOnSJVtvvXUuu+yyFY5781eTzZs3LyeeeGK22Wab9OzZM717986//Mu/5Pe///1yr33yySfzqU99Ko2Njdlggw0yevTo3HPPPct9JdyRRx6Znj175oEHHsiIESPS2NiYYcOGrdL7OWHChFQqlfzxj3/Mpz/96TQ3N6dXr1754he/mFdffTWPPvpoRo4cmcbGxvTv3z9nn332274/AADwXmZHDAAAtEOvvPJK9tprr/z1r3/N1772teywww75/e9/n7POOiszZ87ML3/5y3Tt2jU//vGPs8suu+Soo47KNddck9deey2jR49OtVrND3/4w3Ts2PEtr/G3v/0ts2fPzr/+67+usXkvWrQow4cPz1NPPZWzzjorW221VX75y1/mkEMOWW7sY489lkGDBuXQQw9Nr169MmfOnFx44YX58Ic/nIcffjgbb7xx9ttvv0ycODFf+cpXcsEFF2TnnXdOknzgAx9Ikvz1r3/NkCFDcswxx6S5uTmPPfZYzj333PzTP/1THnjggXTq1ClJMmbMmNx33335xje+ka222iovvPBC7rvvvjz33HO1+VxxxRU54ogjcuCBB+YHP/hBOnXqlIsuuij77LNPbrjhhgwbNizHHHNMnn/++Xz3u9/Ntddem0033TTJ23+F3NSpU/OZz3wmBx54YM4555y0trZmwoQJWbx48TtGr+effz5JMn78+LS0tOSll17Kddddl6FDh+aWW27J0KFDkyQLFy7MXnvtleeffz6TJk3KlltumWnTpq3wfU+SJUuW5IADDsjxxx+fL3/5y3n11VdX6f18w8EHH5zDDz88xx9/fG666aacffbZWbp0aW6++eaceOKJOeWUU3LVVVfltNNOy5ZbbpmDDjrobe8XAADek6oAAMA6b+zYsdUePXrU/v7v//7vapLqj3/84zbjJk2aVE1SvfHGG2vHfvSjH1WTVM8///zqf/7nf1Y7dOjQ5vkVWbp0aXXo0KHVpqam6uOPP/6W4z73uc9VV+X/7LjwwgurSao/+9nP2hw/9thjq0mql1566Vu+9tVXX62+9NJL1R49elS//e1v147/5Cc/qSap3nrrrW977ddee626dOnS6uzZs5ebQ8+ePavjxo17y9cuXLiw2qtXr+rHP/7xNseXLVtW/dCHPlT9yEc+Ujv2rW99q5qkOmvWrLedzxuv79u3b3XnnXeuvvbaa7Xjjz32WLVTp07VLbbYos34JNXx48e/5fleffXV6tKlS6vDhg2rfuITn6gdv+CCC6pJqr/+9a/bjD/++OOXe9/Hjh1bTVL9/ve//7Zzf7v3c/z48dUk1XPOOafNa3bcccdqkuq1115bO7Z06dLqJptsUj3ooIPe9noAAPBe5avJAACgHfrNb36THj165FOf+lSb40ceeWSStPm6rIMPPjgnnHBCvvSlL+XrX/96vvKVr2Tvvfd+y3NXq9UcffTR+f3vf5/LLrss/fr1W2PzvvXWW9PY2JgDDjigzfFRo0YtN/all16q7ZRoaGhIQ0NDevbsmYULF+aRRx5Zqes988wz+exnP5t+/fqloaEhnTp1yhZbbJEkbc7xkY98JFOnTs3Xv/71TJ8+PUuXLm1znjvuuCPPP/98xo4dm1dffbX2eO211zJy5Mjcc889Wbhw4aq+HXn00Ufz1FNPZdSoUalUKrXjW2yxRXbfffeVOsd///d/Z+edd07Xrl1r93jLLbe0ub/bbrstjY2NGTlyZJvXHnbYYW953k9+8pPLHVvZ9/MN+++/f5u/t95661QqlXzsYx+rHWtoaMiWW27Z5ivnAABgXSLEAABAO/Tcc8+lpaWlzb+8T17/rZeGhoY2X6mVJEcddVSWLl2ahoaGnHTSSW953mq1mmOOOSZXXHFFpk6dmgMPPHCNz7tPnz7LHW9paVnu2KhRozJ58uQcc8wxueGGG3L33XfnnnvuySabbJJFixa947Vee+21jBgxItdee21OPfXU3HLLLbn77rszffr0JGlzjh/96EcZO3ZsLrnkkgwZMiS9evXKEUccUfvtmqeffjpJ8qlPfSqdOnVq85g0aVKq1Wrta8JW9f14q/tf0bE3O/fcc3PCCSdk8ODBueaaazJ9+vTcc889GTlyZJv7e6v3fUXHkqR79+5pampqc2xV3s839OrVq83fnTt3Tvfu3dO1a9fljr/yyivveL8AAPBe5DdiAACgHdpoo41y1113pVqttokxzzzzTF599dVsvPHGtWMLFy7MmDFjstVWW+Xpp5/OMccck5/97GfLnfONCHPppZdmypQpOfzww4vM++67717u+BvB4w2tra35xS9+kfHjx+fLX/5y7fjixYtXOng8+OCDuf/++zN16tSMHTu2dvz//u//lhu78cYb5/zzz8/555+fxx9/PNdff32+/OUv55lnnsm0adNq7+d3v/vd7Lbbbiu83ltFjbez0UYbJVn+/t/q2JtdccUVGTp0aC688MI2x1988cXlrrMy7/sb3hz4klV7PwEAYH1iRwwAALRDw4YNy0svvZSf/vSnbY5fdtllteff8NnPfjaPP/54rr322kyZMiXXX399zjvvvDavq1arOfbYY3PppZfmoosuymc+85ki895rr73y4osv5vrrr29z/Kqrrmrzd6VSSbVaTZcuXdocv+SSS7Js2bI2x94Y8+YdGW/EhDef46KLLnrbOb7vfe/L5z//+ey999657777kiR77LFHNthggzz88MPZddddV/jo3Lnz285nRQYNGpRNN900P/zhD1OtVmvHZ8+enTvuuOMdX1+pVJa7vz/+8Y+588472xzbc8898+KLL+bXv/51m+NXX331O17jH6+VrPr7CQAA7Z0dMQAA0A4dccQRueCCCzJ27Ng89thj2X777XP77bdn4sSJ2XfffTN8+PAkr4eLK664Ipdeemm23XbbbLvttvn85z+f0047LXvssUc+8pGPJElOOumkTJkyJUcddVS233772tdNJa//i/eddtqp9vfs2bNzzz33JEn++te/Jkn+53/+J0nSv3//7Lrrrm877/POOy9HHHFEvvGNb2TgwIH51a9+lRtuuKHNuKampnz0ox/Nt771rWy88cbp379/brvttkyZMiUbbLBBm7HbbbddkuR73/teGhsb07Vr1wwYMCAf/OAH84EPfCBf/vKXU61W06tXr/z85z/PTTfd1Ob1ra2t2WuvvTJq1Kh88IMfTGNjY+65555MmzYtBx10UJKkZ8+e+e53v5uxY8fm+eefz6c+9an07t078+bNy/3335958+bVdqVsv/32SZJvf/vbGTt2bDp16pRBgwalsbFxufejQ4cOOfPMM3PMMcfkE5/4RI499ti88MILmTBhwkp9Ndn++++fM888M+PHj8+ee+6ZRx99NGeccUYGDBiQV199tTZu7NixOe+883L44Yfn61//erbccsv8+te/rr3vHTq88/8P38q+nwAAsN6pAgAA67yxY8dWe/To0ebYc889V/3sZz9b3XTTTasNDQ3VLbbYonr66adXX3nllWq1Wq3+8Y9/rHbr1q06duzYNq975ZVXqrvssku1f//+1fnz51er1Wp1iy22qCZZ4WOLLbZo8/pLL730Lce++Vor8uSTT1Y/+clPVnv27FltbGysfvKTn6zecccd1STVSy+9dLlxG264YbWxsbE6cuTI6oMPPljdYostlrvO+eefXx0wYEC1Y8eObc7z8MMPV/fee+9qY2NjdcMNN6x++tOfrj7++OPVJNXx48fX3o/Pfvaz1R122KHa1NRU7datW3XQoEHV8ePHVxcuXNjmOrfddlt1v/32q/bq1avaqVOn6mabbVbdb7/9qj/5yU/ajDv99NOrffv2rXbo0KGapHrrrbe+7XtyySWXVAcOHFjt3Llzdauttqp+//vfr44dO3a59/4f512tVquLFy+unnLKKdXNNtus2rVr1+rOO+9c/elPf7rC1z7++OPVgw46qM37/qtf/aqapPqzn/2sNm5F/117w8q8n9VqtTp+/Phqkuq8efPavP6tzr3nnntWt91227d9jwAA4L2qUq3+w/52AAAA+H8mTpyY//iP/8jjjz+ezTffvN7TAQCAdZKvJgMAACCTJ09O8vpXjC1dujS/+c1v8p3vfCeHH364CAMAAO+CEAMAAEC6d++e8847L4899lgWL16c973vfTnttNPyH//xH/WeGgAArNN8NRkAAAAAAEAhHeo9AQAAAAAAgPZKiAEAAAAAAChEiAEAAAAAACikod4TWFe89tpreeqpp9LY2JhKpVLv6QAAAAAAAHVUrVbz4osvpm/fvunQ4a33vQgxK+mpp55Kv3796j0NAAAAAADgPeSJJ57I5ptv/pbPCzErqbGxMcnrb2hTU1OdZwMAAAAAANTTggUL0q9fv1o/eCtCzEp64+vImpqahBgAAAAAACBJ3vHnTN76S8sAAAAAAAB4V4QYAAAAAACAQoQYAAAAAACAQvxGDAAAAAAA1NGyZcuydOnSek+DN+nYsWMaGhre8Tdg3okQAwAAAAAAdfLSSy/lySefTLVarfdUWIHu3btn0003TefOnVf7HEIMAAAAAADUwbJly/Lkk0+me/fu2WSTTd71zgvWnGq1miVLlmTevHmZNWtWBg4cmA4dVu/XXoQYAAAAAACog6VLl6ZarWaTTTZJt27d6j0d3qRbt27p1KlTZs+enSVLlqRr166rdZ7VyzcAAAAAAMAaYSfMe9fq7oJpc441MA8AAAAAAABWQIgBAAAAAAAoxG/EAAAAAADAe0jla2v3q8qq46tr9XorMnXq1IwbNy4vvPBCvaeyxtkRAwAAAAAArLI77rgjHTt2zMiRI1fpdf3798/555/f5tghhxySP//5z2twdu8dQgwAAAAAALDKvv/97+cLX/hCbr/99jz++OPv6lzdunVL796919DM3luEGAAAAAAAYJUsXLgwP/7xj3PCCSdk//33z9SpU9s8f/3112fXXXdN165ds/HGG+eggw5KkgwdOjSzZ8/Ov//7v6dSqaRSef1r2KZOnZoNNtggSfLoo4+mUqnkT3/6U5tznnvuuenfv3+q1de/Su3hhx/Ovvvum549e6ZPnz4ZM2ZMnn322bI3vhqEGAAAAAAAYJX86Ec/yqBBgzJo0KAcfvjhufTSS2uB5Je//GUOOuig7Lfffvnf//3f3HLLLdl1112TJNdee20233zznHHGGZkzZ07mzJmz3LkHDRqUXXbZJVdeeWWb41dddVVGjRqVSqWSOXPmZM8998yOO+6Ye++9N9OmTcvTTz+dgw8+uPzNr6KGek8AAAAAAABYt0yZMiWHH354kmTkyJF56aWXcsstt2T48OH5xje+kUMPPTRf+9rXauM/9KEPJUl69eqVjh07prGxMS0tLW95/tGjR2fy5Mk588wzkyR//vOfM2PGjFx22WVJkgsvvDA777xzJk6cWHvN97///fTr1y9//vOfs9VWW63xe15ddsQAAAAAAAAr7dFHH83dd9+dQw89NEnS0NCQQw45JN///veTJDNnzsywYcPe1TUOPfTQzJ49O9OnT0+SXHnlldlxxx2zzTbbJElmzJiRW2+9NT179qw9PvjBDyZJ/vrXv76ra69pdsQAAAAAAAArbcqUKXn11Vez2Wab1Y5Vq9V06tQp8+fPT7du3d71NTbddNPstddeueqqq7Lbbrvlhz/8YY4//vja86+99lo+/vGPZ9KkSSt87XuJEAMAAAAAAKyUV199NZdddlnOOeecjBgxos1zn/zkJ3PllVdmhx12yC233JLPfOYzKzxH586ds2zZsne81ujRo3PaaaflsMMOy1//+tfaDpwk2XnnnXPNNdekf//+aWh4b6cOX00GAAAAAACslF/84heZP39+jj766Gy33XZtHp/61KcyZcqUjB8/Pj/84Q8zfvz4PPLII3nggQdy9tln187Rv3///O53v8vf//73PPvss295rYMOOigLFizICSeckL322qvNDpzPfe5zef7553PYYYfl7rvvzt/+9rfceOONOeqoo1Yq8qxN7+1MBAAAAAAA65nq+Gq9p/CWpkyZkuHDh6e5uXm55z75yU9m4sSJaWpqyk9+8pOceeaZ+eY3v5mmpqZ89KMfrY0744wzcvzxx+cDH/hAFi9enGp1xffb1NSUj3/84/nJT35S+/2ZN/Tt2zd/+MMfctppp2WfffbJ4sWLs8UWW2TkyJHp0OG9tQelUn2rO6SNBQsWpLm5Oa2trWlqaqr3dAAAAAAAWMe98sormTVrVgYMGJCuXbvWezqswNut0cp2g/dWFgIAAAAAAGhHhBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAHgPqVTW7mNd9dvf/jaVSiUvvPBCvafytoQYAAAAAABglRx55JH513/917V2vaFDh2bcuHFtju2+++6ZM2dOmpub19o8VkdDvScAAAAAAACwqjp37pyWlpZ6T+Md2REDAAAAAACstqFDh+akk07Kqaeeml69eqWlpSUTJkxoM+bcc8/N9ttvnx49eqRfv3458cQT89JLL7UZ84c//CF77rlnunfvng033DD77LNP5s+fnyOPPDK33XZbvv3tb6dSqaRSqeSxxx5r89Vkra2t6datW6ZNm9bmnNdee2169OhRu9bf//73HHLIIdlwww2z0UYb5cADD8xjjz1W8u0RYgAAAAAAgHfnBz/4QXr06JG77rorZ599ds4444zcdNNNtec7dOiQ73znO3nwwQfzgx/8IL/5zW9y6qmn1p6fOXNmhg0blm233TZ33nlnbr/99nz84x/PsmXL8u1vfztDhgzJsccemzlz5mTOnDnp169fm+s3Nzdnv/32y5VXXtnm+FVXXZUDDzwwPXv2zMsvv5y99torPXv2zO9+97vcfvvt6dmzZ0aOHJklS5YUe298NRkAAAAAAPCu7LDDDhk/fnySZODAgZk8eXJuueWW7L333knS5vddBgwYkDPPPDMnnHBC/uu//itJcvbZZ2fXXXet/Z0k2267be0/d+7cOd27d3/bryIbPXp0jjjiiLz88svp3r17FixYkF/+8pe55pprkiRXX311OnTokEsuuSSVSiVJcumll2aDDTbIb3/724wYMWLNvBlvYkcMAAAAAADwruywww5t/t50003zzDPP1P6+9dZbs/fee2ezzTZLY2NjjjjiiDz33HNZuHBhkv9/R8y7sd9++6WhoSHXX399kuSaa65JY2NjLbDMmDEj//d//5fGxsb07NkzPXv2TK9evfLKK6/kr3/967u69tsRYgAAAAAAgHelU6dObf6uVCp57bXXkiSzZ8/Ovvvum+222y7XXHNNZsyYkQsuuCBJsnTp0iRJt27d3vUcOnfunE996lO56qqrkrz+tWSHHHJIGhpe/3Kw1157LbvssktmzpzZ5vHnP/85o0aNetfXfytCDAAAAAAAUMy9996bV199Neecc0522223bLXVVnnqqafajNlhhx1yyy23vOU5OnfunGXLlr3jtUaPHp1p06bloYceyq233prRo0fXntt5553zl7/8Jb17986WW27Z5tHc3Lz6N/gOhBgAAAAAAKCYD3zgA3n11Vfz3e9+N3/7299y+eWX57//+7/bjDn99NNzzz335MQTT8wf//jH/OlPf8qFF16YZ599NknSv3//3HXXXXnsscfy7LPP1nbbvNmee+6ZPn36ZPTo0enfv39222232nOjR4/OxhtvnAMPPDC///3vM2vWrNx22235t3/7tzz55JPF7r+uIaZ///6pVCrLPT73uc8lSarVaiZMmJC+ffumW7duGTp0aB566KE251i8eHG+8IUvZOONN06PHj1ywAEHLPeGzZ8/P2PGjElzc3Oam5szZsyYvPDCC2vrNgEAAAAAYKVVq2v3UdqOO+6Yc889N5MmTcp2222XK6+8MmeddVabMVtttVVuvPHG3H///fnIRz6SIUOG5Gc/+1nta8VOOeWUdOzYMdtss0022WSTPP744yu8VqVSyWGHHZb777+/zW6YJOnevXt+97vf5X3ve18OOuigbL311jnqqKOyaNGiNDU1lbn5JJVqdW28zSs2b968NluJHnzwwey999659dZbM3To0EyaNCnf+MY3MnXq1Gy11Vb5+te/nt/97nd59NFH09jYmCQ54YQT8vOf/zxTp07NRhttlJNPPjnPP/98ZsyYkY4dOyZJPvaxj+XJJ5/M9773vSTJcccdl/79++fnP//5Ss91wYIFaW5uTmtra9EFAQAAAABg/fDKK69k1qxZGTBgQLp27Vrv6bACb7dGK9sN6hpi3mzcuHH5xS9+kb/85S9Jkr59+2bcuHE57bTTkry++6VPnz6ZNGlSjj/++LS2tmaTTTbJ5ZdfnkMOOSRJ8tRTT6Vfv3751a9+lX322SePPPJIttlmm0yfPj2DBw9OkkyfPj1DhgzJn/70pwwaNGil5ibEAAAAAACwJgkx731rIsS8Z34jZsmSJbniiity1FFHpVKpZNasWZk7d25GjBhRG9OlS5fsueeeueOOO5IkM2bMyNKlS9uM6du3b7bbbrvamDvvvDPNzc21CJMku+22W5qbm2tjVmTx4sVZsGBBmwcAAAAAAMCqeM+EmJ/+9Kd54YUXcuSRRyZJ5s6dmyTp06dPm3F9+vSpPTd37tx07tw5G2644duO6d2793LX6927d23Mipx11lm135Rpbm5Ov379VvveAAAAAACA9dN7JsRMmTIlH/vYx9K3b982xyuVSpu/q9Xqcsfe7M1jVjT+nc5z+umnp7W1tfZ44oknVuY2AAAAAAAAat4TIWb27Nm5+eabc8wxx9SOtbS0JMlyu1aeeeaZ2i6ZlpaWLFmyJPPnz3/bMU8//fRy15w3b95yu23+UZcuXdLU1NTmAQAAAAAAsCreEyHm0ksvTe/evbPffvvVjg0YMCAtLS256aabaseWLFmS2267LbvvvnuSZJdddkmnTp3ajJkzZ04efPDB2pghQ4aktbU1d999d23MXXfdldbW1toYAAAAAACAEhrqPYHXXnstl156acaOHZuGhv9/OpVKJePGjcvEiRMzcODADBw4MBMnTkz37t0zatSoJElzc3OOPvronHzyydloo43Sq1evnHLKKdl+++0zfPjwJMnWW2+dkSNH5thjj81FF12UJDnuuOOy//77Z9CgQWv/hgEAAAAAgPVG3UPMzTffnMcffzxHHXXUcs+deuqpWbRoUU488cTMnz8/gwcPzo033pjGxsbamPPOOy8NDQ05+OCDs2jRogwbNixTp05Nx44da2OuvPLKnHTSSRkxYkSS5IADDsjkyZPL3xwAAAAAALBeq1Sr1Wq9J7EuWLBgQZqbm9Pa2ur3YgAAAAAAeNdeeeWVzJo1KwMGDEjXrl3rPR1W4O3WaGW7Qd13xMA7qVTqd22ZEgAAAABY69b2vxT1L0KL6lDvCQAAAAAAAOuWI488MpVKJd/85jfbHP/pT3+aygpC0qBBg9K5c+f8/e9/X+65oUOHplKp5Oqrr25z/Pzzz0///v3X6LzrQYgBAAAAAABWWdeuXTNp0qTMnz//bcfdfvvteeWVV/LpT386U6dOfctz/cd//EeWLl1aYKb1JcQAAAAAAACrbPjw4WlpaclZZ531tuOmTJmSUaNGZcyYMfn+97+fFf10/WGHHZbW1tZcfPHFpaZbN0IMAAAAAACwyjp27JiJEyfmu9/9bp588skVjnnxxRfzk5/8JIcffnj23nvvLFy4ML/97W+XG9fU1JSvfOUrOeOMM7Jw4cLCM1+7hBgAAAAAAGC1fOITn8iOO+6Y8ePHr/D5q6++OgMHDsy2226bjh075tBDD82UKVNWOPbEE09M165dc+6555ac8lonxAAAAAAAAKtt0qRJ+cEPfpCHH354ueemTJmSww8/vPb34YcfnmuvvTYvvPDCcmO7dOmSM844I9/61rfy7LPPlpzyWiXEAAAAAAAAq+2jH/1o9tlnn3zlK19pc/zhhx/OXXfdlVNPPTUNDQ1paGjIbrvtlkWLFuWHP/zhCs91+OGHp3///vn617++Nqa+VjTUewIAAAAAAMC67Zvf/GZ23HHHbLXVVrVjU6ZMyUc/+tFccMEFbcZefvnlmTJlSk444YTlztOhQ4ecddZZOeigg1b4/LrIjhgAAAAAAOBd2X777TN69Oh897vfTZIsXbo0l19+eQ477LBst912bR7HHHNMZsyYkfvvv3+F59pvv/0yePDgXHTRRWvzFooRYgAAAAAA4L2kWl27jzXkzDPPTPX/ne/666/Pc889l0984hPLjRs4cGC23377TJky5S3PNWnSpLzyyitrbG71VKlW1+C73I4tWLAgzc3NaW1tTVNTU72ns16pVOp3bZ8OAAAAAKCUV155JbNmzcqAAQPStWvXek+HFXi7NVrZbmBHDAAAAAAAQCFCDAAAAAAAQCFCDAAAAAAAQCFCDAAAAAAAQCFCDAAAAAAA1FG1Wq33FHgLa2JthBgAAAAAAKiDjh07JkmWLFlS55nwVl5++eUkSadOnVb7HA1rajIAAAAAAMDKa2hoSPfu3TNv3rx06tQpHTrYO/FeUa1W8/LLL+eZZ57JBhtsUItmq0OIAQAAAACAOqhUKtl0000za9aszJ49u97TYQU22GCDtLS0vKtzCDEAAAAAAFAnnTt3zsCBA3092XtQp06d3tVOmDcIMQAAAAAAUEcdOnRI165d6z0NCvGFcwAAAAAAAIXYEQMAAACwDqhU6nPdarU+1wWA9sKOGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEIa6j0BAACg/iqV+ly3Wq3PdQEAANYWO2IAAAAAAAAKEWIAAAAAAAAKEWIAAAAAAAAKEWIAAAAAAAAKEWIAAAAAAAAKEWIAAAAAAAAKEWIAAAAAAAAKEWIAAAAAAAAKEWIAAAAAAAAKEWIAAAAAAAAKEWIAAAAAAAAKEWIAAAAAAAAKaaj3BAAqlfpdu1qt37UBAAAAgPbPjhgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBCGuo9AQAAAAAAaA8qlfpct1qtz3VZOXbEAAAAAAAAFCLEAAAAAAAAFCLEAAAAAAAAFCLEAAAAAAAAFCLEAAAAAAAAFCLEAAAAAAAAFCLEAAAAAAAAFCLEAAAAAAAAFCLEAAAAAAAAFCLEAAAAAAAAFCLEAAAAAAAAFCLEAAAAAAAAFCLEAAAAAAAAFNJQ7wkAAABQXqVSn+tWq/W5LgAAvFfYEQMAAAAAAFCIEAMAAAAAAFCIEAMAAAAAAFCIEAMAAAAAAFCIEAMAAAAAAFCIEAMAAAAAAFCIEAMAAAAAAFCIEAMAAAAAAFCIEAMAAAAAAFCIEAMAAAAAAFCIEAMAAAAAAFCIEAMAAAAAAFCIEAMAAAAAAFCIEAMAAAAAAFCIEAMAAAAAAFCIEAMAAAAAAFCIEAMAAAAAAFCIEAMAAAAAAFCIEAMAAAAAAFCIEAMAAAAAAFCIEAMAAAAAAFCIEAMAAAAAAFCIEAMAAAAAAFCIEAMAAAAAAFCIEAMAAAAAAFCIEAMAAAAAAFBIQ70nAAAAAAAklUp9rlut1ue6AOsLO2IAAAAAAAAKEWIAAAAAAAAKEWIAAAAAAAAKEWIAAAAAAAAKEWIAAAAAAAAKEWIAAAAAAAAKEWIAAAAAAAAKEWIAAAAAAAAKEWIAAAAAAAAKEWIAAAAAAAAKqXuI+fvf/57DDz88G220Ubp3754dd9wxM2bMqD1frVYzYcKE9O3bN926dcvQoUPz0EMPtTnH4sWL84UvfCEbb7xxevTokQMOOCBPPvlkmzHz58/PmDFj0tzcnObm5owZMyYvvPDC2rhFAAAAAABgPVXXEDN//vzsscce6dSpU37961/n4YcfzjnnnJMNNtigNubss8/Oueeem8mTJ+eee+5JS0tL9t5777z44ou1MePGjct1112Xq6++Orfffnteeuml7L///lm2bFltzKhRozJz5sxMmzYt06ZNy8yZMzNmzJi1ebsAAAAAAMB6plKtVqv1uviXv/zl/OEPf8jvf//7FT5frVbTt2/fjBs3LqeddlqS13e/9OnTJ5MmTcrxxx+f1tbWbLLJJrn88stzyCGHJEmeeuqp9OvXL7/61a+yzz775JFHHsk222yT6dOnZ/DgwUmS6dOnZ8iQIfnTn/6UQYMGveNcFyxYkObm5rS2tqapqWkNvQOsjEqlfteu36dj/WKNAaD+6vXPY/8sXnusMaz7fI7bP2sM6z6f4/XLynaDuu6Iuf7667Prrrvm05/+dHr37p2ddtopF198ce35WbNmZe7cuRkxYkTtWJcuXbLnnnvmjjvuSJLMmDEjS5cubTOmb9++2W677Wpj7rzzzjQ3N9ciTJLstttuaW5uro15s8WLF2fBggVtHgAAAAAAAKuiriHmb3/7Wy688MIMHDgwN9xwQz772c/mpJNOymWXXZYkmTt3bpKkT58+bV7Xp0+f2nNz585N586ds+GGG77tmN69ey93/d69e9fGvNlZZ51V+z2Z5ubm9OvX793dLAAAAAAAsN6pa4h57bXXsvPOO2fixInZaaedcvzxx+fYY4/NhRde2GZc5U37uarV6nLH3uzNY1Y0/u3Oc/rpp6e1tbX2eOKJJ1b2tgAAAAAAAJLUOcRsuumm2Wabbdoc23rrrfP4448nSVpaWpJkuV0rzzzzTG2XTEtLS5YsWZL58+e/7Zinn356uevPmzdvud02b+jSpUuampraPAAAAAAAAFZFXUPMHnvskUcffbTNsT//+c/ZYostkiQDBgxIS0tLbrrpptrzS5YsyW233Zbdd989SbLLLrukU6dObcbMmTMnDz74YG3MkCFD0tramrvvvrs25q677kpra2ttDAAAAAAAwJrWUM+L//u//3t23333TJw4MQcffHDuvvvufO9738v3vve9JK9/ndi4ceMyceLEDBw4MAMHDszEiRPTvXv3jBo1KknS3Nyco48+OieffHI22mij9OrVK6ecckq23377DB8+PMnru2xGjhyZY489NhdddFGS5Ljjjsv++++fQYMG1efmAQAAAACAdq+uIebDH/5wrrvuupx++uk544wzMmDAgJx//vkZPXp0bcypp56aRYsW5cQTT8z8+fMzePDg3HjjjWlsbKyNOe+889LQ0JCDDz44ixYtyrBhwzJ16tR07NixNubKK6/MSSedlBEjRiRJDjjggEyePHnt3SwAAAAAALDeqVSr1Wq9J7EuWLBgQZqbm9Pa2ur3YtaySqV+1/bpWDusMQDUX73+eeyfxWuPNYZ1n89x+2eNYd3nc7x+WdluUNffiAEAAAAAAGjPhBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBCGuo9AQDav0qlfteuVut3bQAAAACwIwYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKCQhnpPAABY91Uq9bt2tVq/awMAAAC8EztiAAAAAAAAChFiAAAAAAAAChFiAAAAAAAAChFiAAAAAAAAChFiAAAAAAAAChFiAAAAAAAAChFiAAAAAAAAChFiAAAAAAAAChFiAAAAAAAAChFiAAAAAAAAChFiAAAAAAAAChFiAAAAAAAAChFiAAAAAAAAChFiAAAAAAAAChFiAAAAAAAAChFiAAAAAAAAChFiAAAAAAAAChFiAAAAAAAACmmo9wQAAHjvq1Tqc91qtT7XBQAAgDXFjhgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBC6hpiJkyYkEql0ubR0tJSe75arWbChAnp27dvunXrlqFDh+ahhx5qc47FixfnC1/4QjbeeOP06NEjBxxwQJ588sk2Y+bPn58xY8akubk5zc3NGTNmTF544YW1cYsAAAAAAMB6rO47YrbddtvMmTOn9njggQdqz5199tk599xzM3ny5Nxzzz1paWnJ3nvvnRdffLE2Zty4cbnuuuty9dVX5/bbb89LL72U/fffP8uWLauNGTVqVGbOnJlp06Zl2rRpmTlzZsaMGbNW7xMAAAAAAFj/NNR9Ag0NbXbBvKFareb888/PV7/61Rx00EFJkh/84Afp06dPrrrqqhx//PFpbW3NlClTcvnll2f48OFJkiuuuCL9+vXLzTffnH322SePPPJIpk2blunTp2fw4MFJkosvvjhDhgzJo48+mkGDBq29mwUAAAAAANYrdd8R85e//CV9+/bNgAEDcuihh+Zvf/tbkmTWrFmZO3duRowYURvbpUuX7LnnnrnjjjuSJDNmzMjSpUvbjOnbt2+222672pg777wzzc3NtQiTJLvttluam5trY1Zk8eLFWbBgQZsHAAAAAADAqqhriBk8eHAuu+yy3HDDDbn44oszd+7c7L777nnuuecyd+7cJEmfPn3avKZPnz615+bOnZvOnTtnww03fNsxvXv3Xu7avXv3ro1ZkbPOOqv2mzLNzc3p16/fu7pXAAAAAABg/VPXEPOxj30sn/zkJ7P99ttn+PDh+eUvf5nk9a8ge0OlUmnzmmq1utyxN3vzmBWNf6fznH766Wltba09nnjiiZW6JwAAAAAAgDfU/avJ/lGPHj2y/fbb5y9/+Uvtd2PevGvlmWeeqe2SaWlpyZIlSzJ//vy3HfP0008vd6158+Ytt9vmH3Xp0iVNTU1tHgAAAAAAAKviPRViFi9enEceeSSbbrppBgwYkJaWltx0002155csWZLbbrstu+++e5Jkl112SadOndqMmTNnTh588MHamCFDhqS1tTV33313bcxdd92V1tbW2hgAAAAAAIASGup58VNOOSUf//jH8773vS/PPPNMvv71r2fBggUZO3ZsKpVKxo0bl4kTJ2bgwIEZOHBgJk6cmO7du2fUqFFJkubm5hx99NE5+eSTs9FGG6VXr1455ZRTal91liRbb711Ro4cmWOPPTYXXXRRkuS4447L/vvvn0GDBtXt3gEAAAAAgPavriHmySefzGGHHZZnn302m2yySXbbbbdMnz49W2yxRZLk1FNPzaJFi3LiiSdm/vz5GTx4cG688cY0NjbWznHeeeeloaEhBx98cBYtWpRhw4Zl6tSp6dixY23MlVdemZNOOikjRoxIkhxwwAGZPHny2r1ZAAAAAABgvVOpVqvVek9iXbBgwYI0NzentbXV78WsZZVK/a7t07F2WOP2zxq3f9a4/avXGlvftccat3/WGNZ9PsftnzWGdZ/P8fplZbvBe+o3YgAAAAAAANoTIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKCQhnpPAAAAAHj3KpX6XLdarc91AdZF/rca1k92xAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABTyngkxZ511ViqVSsaNG1c7Vq1WM2HChPTt2zfdunXL0KFD89BDD7V53eLFi/OFL3whG2+8cXr06JEDDjggTz75ZJsx8+fPz5gxY9Lc3Jzm5uaMGTMmL7zwwlq4KwAAAAAAYH32nggx99xzT773ve9lhx12aHP87LPPzrnnnpvJkyfnnnvuSUtLS/bee++8+OKLtTHjxo3Lddddl6uvvjq33357Xnrppey///5ZtmxZbcyoUaMyc+bMTJs2LdOmTcvMmTMzZsyYtXZ/AAAAAADA+qnuIeall17K6NGjc/HFF2fDDTesHa9Wqzn//PPz1a9+NQcddFC22267/OAHP8jLL7+cq666KknS2tqaKVOm5Jxzzsnw4cOz00475YorrsgDDzyQm2++OUnyyCOPZNq0abnkkksyZMiQDBkyJBdffHF+8Ytf5NFHH63LPQMAAAAAAOuHuoeYz33uc9lvv/0yfPjwNsdnzZqVuXPnZsSIEbVjXbp0yZ577pk77rgjSTJjxowsXbq0zZi+fftmu+22q425884709zcnMGDB9fG7Lbbbmlubq6NWZHFixdnwYIFbR4AAAAAAACroqGeF7/66qtz33335Z577lnuublz5yZJ+vTp0+Z4nz59Mnv27NqYzp07t9lJ88aYN14/d+7c9O7de7nz9+7duzZmRc4666x87WtfW7UbAgAAAAAA+Ad12xHzxBNP5N/+7d9yxRVXpGvXrm85rlKptPm7Wq0ud+zN3jxmRePf6Tynn356Wltba48nnnjiba8JAAAAAADwZnULMTNmzMgzzzyTXXbZJQ0NDWloaMhtt92W73znO2loaKjthHnzrpVnnnmm9lxLS0uWLFmS+fPnv+2Yp59+ernrz5s3b7ndNv+oS5cuaWpqavMAAAAAAABYFXULMcOGDcsDDzyQmTNn1h677rprRo8enZkzZ+b9739/WlpactNNN9Ves2TJktx2223ZfffdkyS77LJLOnXq1GbMnDlz8uCDD9bGDBkyJK2trbn77rtrY+666660trbWxgAAAAAAAJRQt9+IaWxszHbbbdfmWI8ePbLRRhvVjo8bNy4TJ07MwIEDM3DgwEycODHdu3fPqFGjkiTNzc05+uijc/LJJ2ejjTZKr169csopp2T77bfP8OHDkyRbb711Ro4cmWOPPTYXXXRRkuS4447L/vvvn0GDBq3FOwYAAAAAANY3dQsxK+PUU0/NokWLcuKJJ2b+/PkZPHhwbrzxxjQ2NtbGnHfeeWloaMjBBx+cRYsWZdiwYZk6dWo6duxYG3PllVfmpJNOyogRI5IkBxxwQCZPnrzW7wcAAAAAAFi/VKrVarXek1gXLFiwIM3NzWltbfV7MWtZpVK/a/t0rB3WuP2zxu2fNW7/6rXG1nftscbtnzVu/6xx+2eN2z9r3P5Z4/bPGq9fVrYb1O03YgAAAAAAANo7IQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKAQIQYAAAAAAKCQ1QoxixYtyssvv1z7e/bs2Tn//PNz4403rrGJAQAAAAAArOtWK8QceOCBueyyy5IkL7zwQgYPHpxzzjknBx54YC688MI1OkEAAAAAAIB11WqFmPvuuy///M//nCT5n//5n/Tp0yezZ8/OZZddlu985ztrdIIAAAAAAADrqtUKMS+//HIaGxuTJDfeeGMOOuigdOjQIbvttltmz569RicIAAAAAACwrlqtELPlllvmpz/9aZ544onccMMNGTFiRJLkmWeeSVNT0xqdIAAAAAAAwLpqtULMf/7nf+aUU05J//7985GPfCRDhgxJ8vrumJ122mmNThAAAAAAAGBdValWq9XVeeHcuXMzZ86cfOhDH0qHDq/3nLvvvjtNTU354Ac/uEYn+V6wYMGCNDc3p7W11a6ftaxSqd+1V+/Twaqyxu2fNW7/rHH7V681tr5rjzVu/6xx+2eN2z9r3P5Z4/bPGrd/1nj9srLdYLV2xCRJS0tLGhsbc9NNN2XRokVJkg9/+MPtMsIAAAAAAACsjtUKMc8991yGDRuWrbbaKvvuu2/mzJmTJDnmmGNy8sknr9EJAgAAAAAArKtWK8T8+7//ezp16pTHH3883bt3rx0/5JBDMm3atDU2OQAAAAAAgHVZw+q86MYbb8wNN9yQzTffvM3xgQMHZvbs2WtkYgAAAAAAAOu61doRs3DhwjY7Yd7w7LPPpkuXLu96UgAAAAAAAO3BaoWYj370o7nssstqf1cqlbz22mv51re+lb322muNTQ4AAAAAAGBdtlpfTfatb30rQ4cOzb333pslS5bk1FNPzUMPPZTnn38+f/jDH9b0HAEAAAAAANZJq7UjZptttskf//jHfOQjH8nee++dhQsX5qCDDsr//u//5gMf+MCaniMAAAAAAMA6qVKtVqv1nsS6YMGCBWlubk5ra2uamprqPZ31SqVSv2v7dKwd1rj9s8btnzVu/+q1xtZ37bHG7Z81bv+scftnjds/a9z+WeP2zxqvX1a2G6zWjphp06bl9ttvr/19wQUXZMcdd8yoUaMyf/781TklAAAAAABAu7NaIeZLX/pSFixYkCR54IEH8sUvfjH77rtv/va3v+WLX/ziGp0gAAAAAADAuqphdV40a9asbLPNNkmSa665Jh//+MczceLE3Hfffdl3333X6AQBAAAAAADWVau1I6Zz5855+eWXkyQ333xzRowYkSTp1atXbacMAAAAAADA+m61dsT80z/9U774xS9mjz32yN13350f/ehHSZI///nP2XzzzdfoBAEAAAAAANZVq7UjZvLkyWloaMj//M//5MILL8xmm22WJPn1r3+dkSNHrtEJAgAAAAAArKsq1Wq1Wu9JrAsWLFiQ5ubmtLa2pqmpqd7TWa9UKvW7tk/H2mGN2z9r3P5Z4/avXmtsfdcea9z+WeP2zxq3f9a4/bPG7Z81bv+s8fplZbvBan012T9atGhRli5d2uaYUAEAAAAAALCaX022cOHCfP7zn0/v3r3Ts2fPbLjhhm0eAAAAAAAArGaIOfXUU/Ob3/wm//Vf/5UuXbrkkksuyde+9rX07ds3l1122ZqeIwAAAAAAwDpptb6a7Oc//3kuu+yyDB06NEcddVT++Z//OVtuuWW22GKLXHnllRk9evSanicAAAAAAMA6Z7V2xDz//PMZMGBAktd/D+b5559PkvzTP/1Tfve736252QEAAAAAAKzDVivEvP/9789jjz2WJNlmm23y4x//OMnrO2U22GCDNTU3AAAAAACAddpqhZjPfOYzuf/++5Mkp59+eu23YsaNG5cvfelLa3SCAAAAAAAA66pKtVqtvtuTPP7447n33nuz5ZZbZocddlgT83rPWbBgQZqbm9Pa2pqmpqZ6T2e9UqnU79rv/tPByrDG7Z81bv+scftXrzW2vmuPNW7/rHH7Z43bP2vc/lnj9s8at3/WeP2yst1glXbE/OY3v8k222yTBQsWtDn+vve9L8OGDcthhx2W3//+96s3YwAAAAAAgHZmlULM+eefn2OPPXaFZae5uTnHH398zj333DU2OQAAAAAAgHXZKoWY+++/PyNHjnzL50eMGJEZM2a860kBAAAAAAC0B6sUYp5++ul06tTpLZ9vaGjIvHnz3vWkAAAAAAAA2oNVCjGbbbZZHnjggbd8/o9//GM23XTTdz0pAAAAAACA9mCVQsy+++6b//zP/8wrr7yy3HOLFi3K+PHjs//++6+xyQEAAAAAAKzLKtVqtbqyg59++unsvPPO6dixYz7/+c9n0KBBqVQqeeSRR3LBBRdk2bJlue+++9KnT5+Sc66LBQsWpLm5Oa2trWlqaqr3dNYrlUr9rr3ynw7eDWvc/lnj9s8at3/1WmPru/ZY4/bPGrd/1rj9s8btnzVu/6xx+2eN1y8r2w0aVuWkffr0yR133JETTjghp59+et5oOJVKJfvss0/+67/+q11GGAAAAAAAgNWxSiEmSbbYYov86le/yvz58/N///d/qVarGThwYDbccMMS8wMAAAAAAFhnrXKIecOGG26YD3/4w2tyLgAAAAAAAO1Kh3pPAAAAAAAAoL0SYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAoRYgAAAAAAAAqpa4i58MILs8MOO6SpqSlNTU0ZMmRIfv3rX9eer1armTBhQvr27Ztu3bpl6NCheeihh9qcY/HixfnCF76QjTfeOD169MgBBxyQJ598ss2Y+fPnZ8yYMWlubk5zc3PGjBmTF154YW3cIgAAAAAAsB6ra4jZfPPN881vfjP33ntv7r333vzLv/xLDjzwwFpsOfvss3Puuedm8uTJueeee9LS0pK99947L774Yu0c48aNy3XXXZerr746t99+e1566aXsv//+WbZsWW3MqFGjMnPmzEybNi3Tpk3LzJkzM2bMmLV+vwAAAAAAwPqlUq1Wq/WexD/q1atXvvWtb+Woo45K3759M27cuJx22mlJXt/90qdPn0yaNCnHH398Wltbs8kmm+Tyyy/PIYcckiR56qmn0q9fv/zqV7/KPvvsk0ceeSTbbLNNpk+fnsGDBydJpk+fniFDhuRPf/pTBg0atFLzWrBgQZqbm9Pa2pqmpqYyN88KVSr1u/Z769PRflnj9s8at3/WuP2r1xpb37XHGrd/1rj9s8btnzVu/6xx+2eN2z9rvH5Z2W7wnvmNmGXLluXqq6/OwoULM2TIkMyaNStz587NiBEjamO6dOmSPffcM3fccUeSZMaMGVm6dGmbMX379s12221XG3PnnXemubm5FmGSZLfddktzc3NtzIosXrw4CxYsaPMAAAAAAABYFXUPMQ888EB69uyZLl265LOf/Wyuu+66bLPNNpk7d26SpE+fPm3G9+nTp/bc3Llz07lz52y44YZvO6Z3797LXbd37961MSty1lln1X5Tprm5Of369XtX9wkAAAAAAKx/6h5iBg0alJkzZ2b69Ok54YQTMnbs2Dz88MO15ytv2stVrVaXO/Zmbx6zovHvdJ7TTz89ra2ttccTTzyxsrcEAAAAAACQ5D0QYjp37pwtt9wyu+66a84666x86EMfyre//e20tLQkyXK7Vp555pnaLpmWlpYsWbIk8+fPf9sxTz/99HLXnTdv3nK7bf5Rly5d0tTU1OYBAAAAAACwKuoeYt6sWq1m8eLFGTBgQFpaWnLTTTfVnluyZEluu+227L777kmSXXbZJZ06dWozZs6cOXnwwQdrY4YMGZLW1tbcfffdtTF33XVXWltba2MAAAAAAABKaKjnxb/yla/kYx/7WPr165cXX3wxV199dX77299m2rRpqVQqGTduXCZOnJiBAwdm4MCBmThxYrp3755Ro0YlSZqbm3P00Ufn5JNPzkYbbZRevXrllFNOyfbbb5/hw4cnSbbeeuuMHDkyxx57bC666KIkyXHHHZf9998/gwYNqtu9AwAAAAAA7V9dQ8zTTz+dMWPGZM6cOWlubs4OO+yQadOmZe+9906SnHrqqVm0aFFOPPHEzJ8/P4MHD86NN96YxsbG2jnOO++8NDQ05OCDD86iRYsybNiwTJ06NR07dqyNufLKK3PSSSdlxIgRSZIDDjggkydPXrs3CwAAAAAArHcq1Wq1Wu9JrAsWLFiQ5ubmtLa2+r2YtaxSqd+1fTrWDmvc/lnj9s8at3/1WmPru/ZY4/bPGrd/1rj9s8btnzVu/6xx+2eN1y8r2w3ec78RAwAAAAAA0F4IMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAAAIUIMQAAAAAA/197dx9mdV3nf/x1FmQaFMZQYZwVEIwIlVKxDDSl1UgLyWt31cIlDTVLvJlFS60tMQ28y8xYXd2rC3c1F6+rwG42b6gUI0PxhrwJTTe8SSSscLiJBoXz+8N1fjuCWjofDhwej+ua63K+5zvn+z7zmUMxTz7nABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABQixAAAAAAAABRS0xAzbdq0vPe9702vXr3St2/fHHHEEXnsscc6nVOtVjNlypS0tLSksbExo0ePziOPPNLpnPb29px66qnZcccds+2222bcuHH57W9/2+mc5cuXZ8KECWlqakpTU1MmTJiQF154ofRDBAAAAADYJKqpbPQjlcIfwOuqaYiZO3duJk2alPnz52fOnDl56aWXMmbMmKxevbrjnIsvvjiXXXZZpk+fngULFqS5uTkf+tCHsnLlyo5zWltbM3v27MycOTPz5s3LqlWrMnbs2Kxbt67jnPHjx2fhwoW55ZZbcsstt2ThwoWZMGHCJn28AAAAAADA1qVSrVartR7iFc8//3z69u2buXPn5sADD0y1Wk1LS0taW1tz1llnJXl590u/fv1y0UUX5aSTTkpbW1t22mmnXHfddTn66KOTJEuWLEn//v3zox/9KB/+8IezaNGi7L777pk/f37222+/JMn8+fMzcuTIPProoxk6dOgGs7S3t6e9vb3j8xUrVqR///5pa2tL7969N8F3g1fUMqpvPs+O+maN6581rn+l17gaP0S1VqvnsW//pmON6581rn/WuP5Z4/pnjetf3f7dyQ9RB8/jrcuKFSvS1NT0ht1gs3qPmLa2tiRJnz59kiSLFy/O0qVLM2bMmI5zGhoactBBB+Wuu+5Kktx333158cUXO53T0tKSPffcs+OcX/ziF2lqauqIMEny/ve/P01NTR3nvNq0adM6Xsasqakp/fv379oHCwAAAFuQmr3cjZe8AQC2cJtNiKlWq5k8eXIOOOCA7LnnnkmSpUuXJkn69evX6dx+/fp13LZ06dL06NEjb3/721/3nL59+25wzb59+3ac82rnnHNO2traOj6eeeaZt/YAAQAAAACArU73Wg/wilNOOSUPPvhg5s2bt8FtlVf965dqtbrBsVd79TkbO//17qehoSENDQ1/yegAAAAAAAAbtVnsiDn11FPz/e9/P7fffnt22WWXjuPNzc1JssGulWXLlnXskmlubs7atWuzfPny1z3nd7/73QbXff755zfYbQMAAAAAANBVahpiqtVqTjnllMyaNSs//elPM2jQoE63Dxo0KM3NzZkzZ07HsbVr12bu3LkZNWpUkmTEiBHZZpttOp3z3HPP5eGHH+44Z+TIkWlra8s999zTcc7dd9+dtra2jnMAAAAAAAC6Wk1fmmzSpEm54YYb8r3vfS+9evXq2PnS1NSUxsbGVCqVtLa2ZurUqRkyZEiGDBmSqVOnpmfPnhk/fnzHuccff3zOOOOM7LDDDunTp0/OPPPMDB8+PIccckiSZNiwYTn00ENz4okn5uqrr06SfPrTn87YsWMzdOjQ2jx4AAAAAACg7tU0xFx11VVJktGjR3c6PmPGjBx33HFJks9//vNZs2ZNTj755Cxfvjz77bdfbrvttvTq1avj/K9//evp3r17jjrqqKxZsyYHH3xwrr322nTr1q3jnG9/+9s57bTTMmbMmCTJuHHjMn369LIPEAAAAAAA2KpVqtVqtdZDbAlWrFiRpqamtLW1pXfv3rUeZ6tSqdTu2p4dm4Y1rn/WuP6VXuNq/BDVWq2ex779m441rn/WuP753+P653lc/6xx/avbP6v9EHXwPN66/KXdoKbvEQMAAAAAAFDPhBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBCutd6AAAAqKay8Rte43DXXry6CS4CAADA1sqOGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEKEGAAAAAAAgEK613oAAAAAtnzVVDZ+w2sc7tqLVzfBRQAA4M2xIwYAAAAAAKAQIQYAAAAAAKAQL00GAAAAAFuBmr2MpJeQBLZydsQAAAAAAAAUIsQAAAAAAAAUIsQAAAAAAAAUIsQAAAAAAAAUIsQAAAAAAAAUIsQAAAAAAAAUIsQAAAAAAAAUIsQAAAAAAAAUIsQAAAAAAAAUIsQAAAAAAAAUIsQAAAAAAAAUIsQAAAAAAAAU0r3WAwBAV6imsvEbXuNw1124WvgCAAAAAGzJ7IgBAAAAAAAoRIgBAAAAAAAoRIgBAAAAAAAoRIgBAAAAAAAoRIgBAAAAAAAoRIgBAAAAAAAoRIgBAAAAAAAoRIgBAAAAAAAoRIgBAAAAAAAoRIgBAAAAAAAoRIgBAAAAAAAoRIgBAAAAAAAoRIgBAAAAAAAoRIgBAAAAAAAoRIgBAAAAAAAoRIgBAAAAAAAoRIgBAAAAAAAoRIgBAAAAAAAoRIgBAAAAAAAoRIgBAAAAAAAoRIgBAAAAAAAoRIgBAAAAAAAoRIgBAAAAAAAoRIgBAAAAAAAoRIgBAAAAAAAoRIgBAAAAAAAoRIgBAAAAAAAoRIgBAAAAAAAoRIgBAAAAAAAoRIgBAAAAAAAoRIgBAAAAAAAoRIgBAAAAAAAopHutBwDYFKqpbPyG1zjcdReuFr4AAAAAALA5syMGAAAAAACgEDtiAAAAAABgC+BVX7ZMdsQAAAAAAAAUIsQAAAAAAAAUIsQAAAAAAAAUIsQAAAAAAAAUIsQAAAAAAAAUUtMQc+edd+bwww9PS0tLKpVKbrrppk63V6vVTJkyJS0tLWlsbMzo0aPzyCOPdDqnvb09p556anbcccdsu+22GTduXH772992Omf58uWZMGFCmpqa0tTUlAkTJuSFF14o/OgAAAAAAICtXU1DzOrVq/Oe97wn06dP3+jtF198cS677LJMnz49CxYsSHNzcz70oQ9l5cqVHee0trZm9uzZmTlzZubNm5dVq1Zl7NixWbduXcc548ePz8KFC3PLLbfklltuycKFCzNhwoTijw8AAAAAANi6VarVarXWQyRJpVLJ7Nmzc8QRRyR5eTdMS0tLWltbc9ZZZyV5efdLv379ctFFF+Wkk05KW1tbdtppp1x33XU5+uijkyRLlixJ//7986Mf/Sgf/vCHs2jRouy+++6ZP39+9ttvvyTJ/PnzM3LkyDz66KMZOnToRudpb29Pe3t7x+crVqxI//7909bWlt69exf8TvBqlUrtrr15PDvq36ZY42pq9IPkhyiJNd4alF7jmq1vYo3/lzWuf7X6/1y+/ZuO53H9s8b1z5/V9a9un8d+iDpY4/pnjbcuK1asSFNT0xt2g832PWIWL16cpUuXZsyYMR3HGhoactBBB+Wuu+5Kktx333158cUXO53T0tKSPffcs+OcX/ziF2lqauqIMEny/ve/P01NTR3nbMy0adM6Xsqsqakp/fv37+qHCAAAAAAA1LnNNsQsXbo0SdKvX79Ox/v169dx29KlS9OjR4+8/e1vf91z+vbtu8H99+3bt+OcjTnnnHPS1tbW8fHMM8+8pccDAAAAAABsfbrXeoA3UnnVXq5qtbrBsVd79TkbO/+N7qehoSENDQ1/5bQAAAAAAAD/32a7I6a5uTlJNti1smzZso5dMs3NzVm7dm2WL1/+uuf87ne/2+D+n3/++Q122wAAAAAAAHSlzTbEDBo0KM3NzZkzZ07HsbVr12bu3LkZNWpUkmTEiBHZZpttOp3z3HPP5eGHH+44Z+TIkWlra8s999zTcc7dd9+dtra2jnMAAAAAAABKqOlLk61atSpPPPFEx+eLFy/OwoUL06dPnwwYMCCtra2ZOnVqhgwZkiFDhmTq1Knp2bNnxo8fnyRpamrK8ccfnzPOOCM77LBD+vTpkzPPPDPDhw/PIYcckiQZNmxYDj300Jx44om5+uqrkySf/vSnM3bs2AwdOnTTP2gAAAAAAGCrUdMQc++99+aDH/xgx+eTJ09Okhx77LG59tpr8/nPfz5r1qzJySefnOXLl2e//fbLbbfdll69enV8zde//vV07949Rx11VNasWZODDz441157bbp169Zxzre//e2cdtppGTNmTJJk3LhxmT59+iZ6lAAAAAAAwNaqUq1Wq7UeYkuwYsWKNDU1pa2tLb179671OFuVSqV21/bs2DQ2xRpXU6MfJD9ESazx1qD0GtdsfRNr/L+scf2r1f/n8u3fdDyP6581rn/+rK5/dfs89kPUwRrXP2u8dflLu8Fm+x4xAAAAAAAAWzohBgAAAAAAoJCavkcMAAAAbC0q55V+KREvGQIAsDmyIwYAAAAAAKAQIQYAAAAAAKAQL00GAAAAABAvIwmUYUcMAAAAAABAIUIMAAAAAABAIUIMAAAAAABAIUIMAAAAAABAIUIMAAAAAABAIUIMAAAAAABAId1rPQAAAFD/qqls/IbXONy1F69ugosAAABsnB0xAAAAAAAAhQgxAAAAAAAAhQgxAAAAAAAAhQgxAAAAAAAAhQgxAAAAAAAAhQgxAAAAAAAAhQgxAAAAAAAAhQgxAAAAAAAAhQgxAAAAAAAAhQgxAAAAAAAAhQgxAAAAAAAAhQgxAAAAAAAAhXSv9QAAALx1lfMqha9QLXz/AAAAUJ/siAEAAAAAAChEiAEAAAAAAChEiAEAAAAAACjEe8QA4L0lAAAAAKAQO2IAAAAAAAAKsSMGAAAAgFTzGjvli2+gt4MegPpmRwwAAAAAAEAhQgwAAAAAAEAhQgwAAAAAAEAhQgwAAAAAAEAhQgwAAAAAAEAhQgwAAAAAAEAhQgwAAAAAAEAhQgwAAAAAAEAhQgwAAAAAAEAhQgwAAAAAAEAhQgwAAAAAAEAhQgwAAAAAAEAhQgwAAAAAAEAhQgwAAAAAAEAhQgwAAAAAAEAhQgwAAAAAAEAhQgwAAAAAAEAhQgwAAAAAAEAhQgwAAAAAAEAhQgwAAAAAAEAhQgwAAAAAAEAhQgwAAAAAAEAhQgwAAAAAAEAhQgwAAAAAAEAhQgwAAAAAAEAhQgwAAAAAAEAhQgwAAAAAAEAhQgwAAAAAAEAhQgwAAAAAAEAh3Ws9AGwOqqls/IbXONx1F64WvgAAAAAAALVkRwwAAAAAAEAhQgwAAAAAAEAhQgwAAAAAAEAhQgwAAAAAAEAhQgwAAAAAAEAhQgwAAAAAAEAhQgwAAAAAAEAhQgwAAAAAAEAhQgwAAAAAAEAhQgwAAAAAAEAhQgwAAAAAAEAhQgwAAAAAAEAhQgwAAAAAAEAhQgwAAAAAAEAhQgwAAAAAAEAhQgwAAAAAAEAhQgwAAAAAAEAhQgwAAAAAAEAhQgwAAAAAAEAhQgwAAAAAAEAhQgwAAAAAAEAhQgwAAAAAAEAh3Ws9AAAAAAAAbAqV8yqFr1AtfP9sieyIAQAAAAAAKESIAQAAAAAAKESIAQAAAAAAKESIAQAAAAAAKESIAQAAAAAAKESIAQAAAAAAKESIAQAAAAAAKKR7rQcAAAAAgC1B5bxK4StUC98/ALVgRwwAAAAAAEAhQgwAAAAAAEAhXpoMAAC2AF4KBQAAYMtkRwwAAAAAAEAhQgwAAAAAAEAhQgwAAAAAAEAhQgwAAAAAAEAhQgwAAAAAAEAhQgwAAAAAAEAh3Ws9AFu+ynmVwleoFr5/AAAAAAAoQ4gBgK2AaA4AAABQG16aDAAAAAAAoBAhBgAAAAAAoBAhBgAAAAAAoBAhBgAAAAAAoBAhBgAAAAAAoJDutR4AAAAAoB5UzqsUvkK18P0DACXYEQMAAAAAAFCIEAMAAAAAAFCIEAMAAAAAAFCI94gB3pDXOQYAAAAAeHPsiAEAAAAAAChEiAEAAAAAAChEiAEAAAAAAChEiAEAAAAAAChEiAEAAAAAAChEiAEAAAAAAChEiAEAAAAAACike60HAAAAIKmcVyl8hWrh+wcAADbGjhgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBChBgAAAAAAIBCtqoQc+WVV2bQoEF529velhEjRuRnP/tZrUcCAAAAAADq2FYTYm688ca0trbmi1/8Yh544IF84AMfyGGHHZann3661qMBAAAAAAB1aqsJMZdddlmOP/74nHDCCRk2bFguv/zy9O/fP1dddVWtRwMAAAAAAOpU91oPsCmsXbs29913X84+++xOx8eMGZO77rpro1/T3t6e9vb2js/b2tqSJCtWrCg36Jbqz6UvUP57XrNV3VJ+nqzxW7iwNX6ZNa65LXyNa/pdtsb/yxrXnDV+Cxe3xi+zxjVnjd/Cxa3xy+p0jbeU9U2s8Zu+sDX+/6xxzVnjN3nhLWiNN6FXekG1Wn3d87aKEPP73/8+69atS79+/Tod79evX5YuXbrRr5k2bVrOO++8DY7379+/yIy8nqY6uMJrXbhmV97MWOP6Z43rX9nvQ02/y9b4f1nj+meN6581rn/WuP7V6Rpb3//DGtc/a1z/rPHWaOXKlWl6ne/RVhFiXlGpVDp9Xq1WNzj2inPOOSeTJ0/u+Hz9+vX54x//mB122OE1v4ausWLFivTv3z/PPPNMevfuXetxKMAa1z9rXP+scf2zxvXPGtc/a1z/rHH9s8b1zxrXP2tc/6zx1q1arWblypVpaWl53fO2ihCz4447plu3bhvsflm2bNkGu2Re0dDQkIaGhk7Htt9++1IjshG9e/f2h1eds8b1zxrXP2tc/6xx/bPG9c8a1z9rXP+scf2zxvXPGtc/a7z1er2dMK/4m00wR8316NEjI0aMyJw5czodnzNnTkaNGlWjqQAAAAAAgHq3VeyISZLJkydnwoQJ2XfffTNy5Mhcc801efrpp/OZz3ym1qMBAAAAAAB1aqsJMUcffXT+8Ic/5Ctf+Uqee+657LnnnvnRj36UgQMH1no0XqWhoSHnnnvuBi8NR/2wxvXPGtc/a1z/rHH9s8b1zxrXP2tc/6xx/bPG9c8a1z9rzF+iUq1Wq7UeAgAAAAAAoB5tFe8RAwAAAAAAUAtCDAAAAAAAQCFCDAAAAAAAQCFCDAAAAAAAQCFCDJuVO++8M4cffnhaWlpSqVRy00031XokutC0adPy3ve+N7169Urfvn1zxBFH5LHHHqv1WHShq666Ku9+97vTu3fv9O7dOyNHjszNN99c67EoaNq0aalUKmltba31KHSRKVOmpFKpdPpobm6u9Vh0sWeffTb/9E//lB122CE9e/bMXnvtlfvuu6/WY9FFdt111w2ex5VKJZMmTar1aHSRl156Kf/yL/+SQYMGpbGxMYMHD85XvvKVrF+/vtaj0UVWrlyZ1tbWDBw4MI2NjRk1alQWLFhQ67F4C97o9x3VajVTpkxJS0tLGhsbM3r06DzyyCO1GZY35Y3WeNasWfnwhz+cHXfcMZVKJQsXLqzJnLx5r7fGL774Ys4666wMHz482267bVpaWvLJT34yS5Ysqd3AbFaEGDYrq1evznve855Mnz691qNQwNy5czNp0qTMnz8/c+bMyUsvvZQxY8Zk9erVtR6NLrLLLrvkwgsvzL333pt77703f/d3f5ePfexj/gJRpxYsWJBrrrkm7373u2s9Cl1sjz32yHPPPdfx8dBDD9V6JLrQ8uXLs//++2ebbbbJzTffnF/96lf52te+lu23377Wo9FFFixY0Ok5PGfOnCTJkUceWePJ6CoXXXRR/u3f/i3Tp0/PokWLcvHFF+eSSy7JN7/5zVqPRhc54YQTMmfOnFx33XV56KGHMmbMmBxyyCF59tlnaz0ab9Ib/b7j4osvzmWXXZbp06dnwYIFaW5uzoc+9KGsXLlyE0/Km/VGa7x69ersv//+ufDCCzfxZHSV11vjP/3pT7n//vvzpS99Kffff39mzZqVX//61xk3blwNJmVzVKlWq9VaDwEbU6lUMnv27BxxxBG1HoVCnn/++fTt2zdz587NgQceWOtxKKRPnz655JJLcvzxx9d6FLrQqlWrss8+++TKK6/MBRdckL322iuXX355rceiC0yZMiU33XSTf6FXx84+++z8/Oc/z89+9rNaj8Im0tramh/+8Id5/PHHU6lUaj0OXWDs2LHp169fvvWtb3Uc+4d/+If07Nkz1113XQ0noyusWbMmvXr1yve+97189KMf7Ti+1157ZezYsbngggtqOB1d4dW/76hWq2lpaUlra2vOOuusJEl7e3v69euXiy66KCeddFINp+XNeL3faT355JMZNGhQHnjggey1116bfDa6xl/ye8sFCxbkfe97X5566qkMGDBg0w3HZsmOGKBm2trakrz8i3rqz7p16zJz5sysXr06I0eOrPU4dLFJkyblox/9aA455JBaj0IBjz/+eFpaWjJo0KB8/OMfz29+85taj0QX+v73v5999903Rx55ZPr27Zu99947//7v/17rsShk7dq1uf766zNx4kQRpo4ccMAB+clPfpJf//rXSZJf/vKXmTdvXj7ykY/UeDK6wksvvZR169blbW97W6fjjY2NmTdvXo2moqTFixdn6dKlGTNmTMexhoaGHHTQQbnrrrtqOBnwVrS1taVSqdh5TpKke60HALZO1Wo1kydPzgEHHJA999yz1uPQhR566KGMHDkyf/7zn7Pddttl9uzZ2X333Ws9Fl1o5syZuf/++71OeZ3ab7/98p//+Z955zvfmd/97ne54IILMmrUqDzyyCPZYYcdaj0eXeA3v/lNrrrqqkyePDlf+MIXcs899+S0005LQ0NDPvnJT9Z6PLrYTTfdlBdeeCHHHXdcrUehC5111llpa2vLu971rnTr1i3r1q3LV7/61XziE5+o9Wh0gV69emXkyJE5//zzM2zYsPTr1y//9V//lbvvvjtDhgyp9XgUsHTp0iRJv379Oh3v169fnnrqqVqMBLxFf/7zn3P22Wdn/Pjx6d27d63HYTMgxAA1ccopp+TBBx/0L7rq0NChQ7Nw4cK88MIL+e53v5tjjz02c+fOFWPqxDPPPJPTTz89t9122wb/SpP6cNhhh3X89/DhwzNy5Mjstttu+Y//+I9Mnjy5hpPRVdavX5999903U6dOTZLsvffeeeSRR3LVVVcJMXXoW9/6Vg477LC0tLTUehS60I033pjrr78+N9xwQ/bYY48sXLgwra2taWlpybHHHlvr8egC1113XSZOnJi//du/Tbdu3bLPPvtk/Pjxuf/++2s9GgW9euditVq1mxG2QC+++GI+/vGPZ/369bnyyitrPQ6bCSEG2OROPfXUfP/738+dd96ZXXbZpdbj0MV69OiRd7zjHUmSfffdNwsWLMg3vvGNXH311TWejK5w3333ZdmyZRkxYkTHsXXr1uXOO+/M9OnT097enm7dutVwQrratttum+HDh+fxxx+v9Sh0kZ133nmDOD5s2LB897vfrdFElPLUU0/lxz/+cWbNmlXrUehin/vc53L22Wfn4x//eJKXw/lTTz2VadOmCTF1YrfddsvcuXOzevXqrFixIjvvvHOOPvroDBo0qNajUUBzc3OSl3fG7Lzzzh3Hly1btsEuGWDz9uKLL+aoo47K4sWL89Of/tRuGDp4jxhgk6lWqznllFMya9as/PSnP/WXiK1EtVpNe3t7rcegixx88MF56KGHsnDhwo6PfffdN8ccc0wWLlwowtSh9vb2LFq0qNMvBdiy7b///nnsscc6Hfv1r3+dgQMH1mgiSpkxY0b69u3b6c2+qQ9/+tOf8jd/0/mv8926dcv69etrNBGlbLvtttl5552zfPny3HrrrfnYxz5W65EoYNCgQWlubs6cOXM6jq1duzZz587NqFGjajgZ8Nd4JcI8/vjj+fGPf+ylnenEjhg2K6tWrcoTTzzR8fnixYuzcOHC9OnTJwMGDKjhZHSFSZMm5YYbbsj3vve99OrVq+N1cJuamtLY2Fjj6egKX/jCF3LYYYelf//+WblyZWbOnJk77rgjt9xyS61Ho4v06tVrg/d12nbbbbPDDjt4v6c6ceaZZ+bwww/PgAEDsmzZslxwwQVZsWKFf2FdR/75n/85o0aNytSpU3PUUUflnnvuyTXXXJNrrrmm1qPRhdavX58ZM2bk2GOPTffu/tpXbw4//PB89atfzYABA7LHHnvkgQceyGWXXZaJEyfWejS6yK233ppqtZqhQ4fmiSeeyOc+97kMHTo0n/rUp2o9Gm/SG/2+o7W1NVOnTs2QIUMyZMiQTJ06NT179sz48eNrODV/jTda4z/+8Y95+umns2TJkiTp+Icxzc3NHbui2Ly93hq3tLTkH//xH3P//ffnhz/8YdatW9fxe68+ffqkR48etRqbzUUVNiO33357NckGH8cee2ytR6MLbGxtk1RnzJhR69HoIhMnTqwOHDiw2qNHj+pOO+1UPfjgg6u33XZbrceisIMOOqh6+umn13oMusjRRx9d3XnnnavbbLNNtaWlpfr3f//31UceeaTWY9HFfvCDH1T33HPPakNDQ/Vd73pX9Zprrqn1SHSxW2+9tZqk+thjj9V6FApYsWJF9fTTT68OGDCg+ra3va06ePDg6he/+MVqe3t7rUeji9x4443VwYMHV3v06FFtbm6uTpo0qfrCCy/Ueizegjf6fcf69eur5557brW5ubna0NBQPfDAA6sPPfRQbYfmr/JGazxjxoyN3n7uuefWdG7+cq+3xosXL37N33vdfvvttR6dzUClWq1WS4YeAAAAAACArZX3iAEAAAAAAChEiAEAAAAAAChEiAEAAAAAAChEiAEAAAAAAChEiAEAAAAAAChEiAEAAAAAAChEiAEAAAAAAChEiAEAAAAAAChEiAEAANiMPfnkk6lUKlm4cGGtRwEAAN4EIQYAANhsVCqV1/047rjj3vR9P/nkkzn++OMzaNCgNDY2Zrfddsu5556btWvXdjrv9NNPz4gRI9LQ0JC99trrr7rGtddem+233/5NzwgAANSf7rUeAAAA4BXPPfdcx3/feOON+fKXv5zHHnus41hjY+Obvu9HH30069evz9VXX513vOMdefjhh3PiiSdm9erVufTSSzvOq1armThxYu6+++48+OCDb/p6AAAAiR0xAADAZqS5ubnjo6mpKZVKpdOxG264Ibvttlt69OiRoUOH5rrrruv42okTJ+bd73532tvbkyQvvvhiRowYkWOOOSZJcuihh2bGjBkZM2ZMBg8enHHjxuXMM8/MrFmzOs1wxRVXZNKkSRk8ePBfNfsdd9yRT33qU2lra+vYwTNlypQkyfXXX5999903vXr1SnNzc8aPH59ly5Z1fO3y5ctzzDHHZKeddkpjY2OGDBmSGTNmbPQ669evz4knnph3vvOdeeqpp5IkU6ZMyYABA9LQ0JCWlpacdtppf9XsAABAOUIMAACwRZg9e3ZOP/30nHHGGXn44Ydz0kkn5VOf+lRuv/32JC8HlNWrV+fss89OknzpS1/K73//+1x55ZWveZ9tbW3p06dPl8w3atSoXH755endu3eee+65PPfccznzzDOTJGvXrs3555+fX/7yl7npppuyePHiTi+z9qUvfSm/+tWvcvPNN2fRokW56qqrsuOOO25wjbVr1+aoo47Kvffem3nz5mXgwIH5zne+k69//eu5+uqr8/jjj+emm27K8OHDu+QxAQAAb52XJgMAALYIl156aY477ricfPLJSZLJkydn/vz5ufTSS/PBD34w2223Xa6//vocdNBB6dWrV772ta/lJz/5SZqamjZ6f//zP/+Tb37zm/na177WJfP16NGj0y6e/2vixIkd/z148OBcccUVed/73pdVq1Zlu+22y9NPP5299947++67b5Jk11133eD+V61alY9+9KNZs2ZN7rjjjo7H9fTTT6e5uTmHHHJIttlmmwwYMCDve9/7uuQxAQAAb50dMQAAwBZh0aJF2X///Tsd23///bNo0aKOz0eOHJkzzzwz559/fs4444wceOCBG72vJUuW5NBDD82RRx6ZE044oejcSfLAAw/kYx/7WAYOHJhevXpl9OjRSV6OKEny2c9+NjNnzsxee+2Vz3/+87nrrrs2uI9PfOITWbVqVW677bZOcenII4/MmjVrMnjw4Jx44omZPXt2XnrppeKPCQAA+MsIMQAAwBajUql0+rxarXY6tn79+vz85z9Pt27d8vjjj2/0PpYsWZIPfvCDGTlyZK655pqi8ybJ6tWrM2bMmI4dOwsWLMjs2bOTvPxSY0ly2GGH5amnnkpra2uWLFmSgw8+uONlzV7xkY98JA8++GDmz5/f6Xj//v3z2GOP5V//9V/T2NiYk08+OQceeGBefPHF4o8NAAB4Y0IMAACwRRg2bFjmzZvX6dhdd92VYcOGdXx+ySWXZNGiRZk7d25uvfXWDd7w/tlnn83o0aOzzz77ZMaMGfmbv+navxL16NEj69at63Ts0Ucfze9///tceOGF+cAHPpB3vetdWbZs2QZfu9NOO+W4447L9ddfn8svv3yDSPTZz342F154YcaNG5e5c+d2uq2xsTHjxo3LFVdckTvuuCO/+MUv8tBDD3XpYwMAAN4c7xEDAABsET73uc/lqKOOyj777JODDz44P/jBDzJr1qz8+Mc/TpIsXLgwX/7yl/Od73wn+++/f77xjW/k9NNPz0EHHZTBgwdnyZIlGT16dAYMGJBLL700zz//fMd9/9/3dHniiSeyatWqLF26NGvWrMnChQuTJLvvvnt69OjxujPuuuuuWbVqVX7yk5/kPe95T3r27JkBAwakR48e+eY3v5nPfOYzefjhh3P++ed3+rovf/nLGTFiRPbYY4+0t7fnhz/8YafA9IpTTz0169aty9ixY3PzzTfngAMOyLXXXpt169Zlv/32S8+ePXPdddelsbExAwcOfLPfagAAoAsJMQAAwBbhiCOOyDe+8Y1ccsklOe200zJo0KDMmDEjo0ePzp///Occc8wxOe6443L44YcnSY4//vj893//dyZMmJA777wzt912W5544ok88cQT2WWXXTrdd7Va7fjvE044odOOk7333jtJsnjx4uy6666vO+OoUaPymc98JkcffXT+8Ic/5Nxzz82UKVNy7bXX5gtf+EKuuOKK7LPPPrn00kszbty4jq/r0aNHzjnnnDz55JNpbGzMBz7wgcycOXOj12htbc369evzkY98JLfccku23377XHjhhZk8eXLWrVuX4cOH5wc/+EF22GGHv+r7CwAAlFGp/t+/cQAAAAAAANBlvEcMAAAAAABAIUIMAADAX+iwww7Ldtttt9GPqVOn1no8AABgM+SlyQAAAP5Czz77bNasWbPR2/r06ZM+ffps4okAAIDNnRADAAAAAABQiJcmAwAAAAAAKESIAQAAAAAAKESIAQAAAAAAKESIAQAAAAAAKESIAQAAAAAAKESIAQAAAAAAKESIAQAAAAAAKOT/AQ/1lJyZSAPmAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 2000x1500 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Importing the matplotlib library\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "# Declaring the figure or the plot (y, x) or (width, height)\n",
    "plt.figure(figsize=[20, 15])\n",
    "\n",
    "X = np.arange(1,len(tox21_tasks)+1)\n",
    "plt.bar(X + 0.2, one, color = 'g', width = 0.25)\n",
    "plt.bar(X + 0.4, zero, color = 'b', width = 0.25)\n",
    "plt.bar(X + 0.6, nan, color = 'r', width = 0.25)\n",
    "\n",
    "# Creating the legend of the bars in the plot\n",
    "plt.legend(['Active' , 'Inactive' ,'NAN'])\n",
    "# Overiding the x axis with the country names\n",
    "plt.xticks([i + 0.25 for i in range(1,13)], X)\n",
    "# Giving the tilte for the plot\n",
    "plt.title(\"Tox21 dataset diagram\")\n",
    "# Namimg the x and y axis\n",
    "plt.xlabel('Tox21_tasks')\n",
    "plt.ylabel('Cases')\n",
    "# Saving the plot as a 'png'\n",
    "plt.savefig('4BarPlot.png')\n",
    "# Displaying the bar plot\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "id": "6grIE_JeqkUZ"
   },
   "source": [
    "# Required functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "hidden": true,
    "id": "IzllOg474i99"
   },
   "outputs": [],
   "source": [
    "from dgllife.model import MLPPredictor\n",
    "\n",
    "def create_dataset_with_gcn(dataset, class_embed_vector, GCN, tasks, numberTask):\n",
    "\n",
    "    created_data = []\n",
    "    data = np.arange(len(tasks))\n",
    "    onehot_encoded = to_categorical(data)\n",
    "\n",
    "    for i, data in enumerate(dataset):\n",
    "        smiles, g, label, mask = data\n",
    "        g = g.to(device)\n",
    "        g = dgl.add_self_loop(g)\n",
    "        graph_feats = g.ndata.pop('h')\n",
    "        embbed = GCN(g, graph_feats)\n",
    "        embbed = embbed.to('cpu')\n",
    "        embbed = embbed.detach().numpy()\n",
    "        a = ( embbed, onehot_encoded[numberTask], class_embed_vector[numberTask], label, numberTask, tasks[numberTask])\n",
    "        created_data.append(a)\n",
    "    print('Data created!!')\n",
    "    return created_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# Calculation of embedded vectors for each class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NR-AR=> positive: 309 - negative: 6956\n",
      "NR-AR-LBD=> positive: 237 - negative: 6521\n",
      "NR-AhR=> positive: 768 - negative: 5781\n",
      "NR-Aromatase=> positive: 300 - negative: 5521\n",
      "NR-ER=> positive: 793 - negative: 5400\n",
      "NR-ER-LBD=> positive: 350 - negative: 6605\n",
      "NR-PPAR-gamma=> positive: 186 - negative: 6264\n",
      "SR-ARE=> positive: 942 - negative: 4890\n",
      "SR-ATAD5=> positive: 264 - negative: 6808\n",
      "SR-HSE=> positive: 372 - negative: 6095\n",
      "SR-MMP=> positive: 918 - negative: 4892\n",
      "SR-p53=> positive: 423 - negative: 6351\n"
     ]
    }
   ],
   "source": [
    "df_positive, df_negative = separate_active_and_inactive_data(df, tox21_tasks)\n",
    "\n",
    "for i,d in enumerate(zip(df_positive,df_negative)):\n",
    "    print(f'{tox21_tasks[i]}=> positive: {len(d[0])} - negative: {len(d[1])}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing dgl graphs from scratch...\n",
      "Processing dgl graphs from scratch...\n",
      "Processing dgl graphs from scratch...\n",
      "Processing dgl graphs from scratch...\n",
      "Processing dgl graphs from scratch...\n",
      "Processing dgl graphs from scratch...\n",
      "Processing dgl graphs from scratch...\n",
      "Processing dgl graphs from scratch...\n",
      "Processing dgl graphs from scratch...\n",
      "Processing dgl graphs from scratch...\n",
      "Processing dgl graphs from scratch...\n",
      "Processing dgl graphs from scratch...\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/6956\n",
      "Processing molecule 2000/6956\n",
      "Processing molecule 3000/6956\n",
      "Processing molecule 4000/6956\n",
      "Processing molecule 5000/6956\n",
      "Processing molecule 6000/6956\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/6521\n",
      "Processing molecule 2000/6521\n",
      "Processing molecule 3000/6521\n",
      "Processing molecule 4000/6521\n",
      "Processing molecule 5000/6521\n",
      "Processing molecule 6000/6521\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/5781\n",
      "Processing molecule 2000/5781\n",
      "Processing molecule 3000/5781\n",
      "Processing molecule 4000/5781\n",
      "Processing molecule 5000/5781\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/5521\n",
      "Processing molecule 2000/5521\n",
      "Processing molecule 3000/5521\n",
      "Processing molecule 4000/5521\n",
      "Processing molecule 5000/5521\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/5400\n",
      "Processing molecule 2000/5400\n",
      "Processing molecule 3000/5400\n",
      "Processing molecule 4000/5400\n",
      "Processing molecule 5000/5400\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/6605\n",
      "Processing molecule 2000/6605\n",
      "Processing molecule 3000/6605\n",
      "Processing molecule 4000/6605\n",
      "Processing molecule 5000/6605\n",
      "Processing molecule 6000/6605\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/6264\n",
      "Processing molecule 2000/6264\n",
      "Processing molecule 3000/6264\n",
      "Processing molecule 4000/6264\n",
      "Processing molecule 5000/6264\n",
      "Processing molecule 6000/6264\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/4890\n",
      "Processing molecule 2000/4890\n",
      "Processing molecule 3000/4890\n",
      "Processing molecule 4000/4890\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/6808\n",
      "Processing molecule 2000/6808\n",
      "Processing molecule 3000/6808\n",
      "Processing molecule 4000/6808\n",
      "Processing molecule 5000/6808\n",
      "Processing molecule 6000/6808\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/6095\n",
      "Processing molecule 2000/6095\n",
      "Processing molecule 3000/6095\n",
      "Processing molecule 4000/6095\n",
      "Processing molecule 5000/6095\n",
      "Processing molecule 6000/6095\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/4892\n",
      "Processing molecule 2000/4892\n",
      "Processing molecule 3000/4892\n",
      "Processing molecule 4000/4892\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/6351\n",
      "Processing molecule 2000/6351\n",
      "Processing molecule 3000/6351\n",
      "Processing molecule 4000/6351\n",
      "Processing molecule 5000/6351\n",
      "Processing molecule 6000/6351\n"
     ]
    }
   ],
   "source": [
    "dataset_positive = [DATASET(d,smiles_to_bigraph, AttentiveFPAtomFeaturizer(), cache_file_path = cache_path) for d in df_positive]\n",
    "dataset_negative = [DATASET(d,smiles_to_bigraph, AttentiveFPAtomFeaturizer(), cache_file_path = cache_path) for d in df_negative]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "class vector created!!\n"
     ]
    }
   ],
   "source": [
    "embed_class_tox21 = get_embedding_vector_class(dataset_positive, dataset_negative, radius=2, size = 512)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "id": "zIKQi__XAcia"
   },
   "source": [
    "# Classification with BioAct-Het and AttentiveFp GCN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "hidden": true,
    "id": "lDS5UguKr_x_",
    "outputId": "da58be7e-197e-4838-f5f1-a0b2d6b87cb2"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading GCN_attentivefp_Tox21_pre_trained.pth from https://data.dgl.ai/dgllife/pre_trained/gcn_attentivefp_tox21.pth...\n",
      "Pretrained model loaded\n"
     ]
    }
   ],
   "source": [
    "model_name = 'GCN_attentivefp_Tox21'\n",
    "gcn_model = get_tox21_model(model_name)\n",
    "gcn_model.eval()\n",
    "gcn_model = gcn_model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "hidden": true,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/7265\n",
      "Processing molecule 2000/7265\n",
      "Processing molecule 3000/7265\n",
      "Processing molecule 4000/7265\n",
      "Processing molecule 5000/7265\n",
      "Processing molecule 6000/7265\n",
      "Processing molecule 7000/7265\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/6758\n",
      "Processing molecule 2000/6758\n",
      "Processing molecule 3000/6758\n",
      "Processing molecule 4000/6758\n",
      "Processing molecule 5000/6758\n",
      "Processing molecule 6000/6758\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/6549\n",
      "Processing molecule 2000/6549\n",
      "Processing molecule 3000/6549\n",
      "Processing molecule 4000/6549\n",
      "Processing molecule 5000/6549\n",
      "Processing molecule 6000/6549\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/5821\n",
      "Processing molecule 2000/5821\n",
      "Processing molecule 3000/5821\n",
      "Processing molecule 4000/5821\n",
      "Processing molecule 5000/5821\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/6193\n",
      "Processing molecule 2000/6193\n",
      "Processing molecule 3000/6193\n",
      "Processing molecule 4000/6193\n",
      "Processing molecule 5000/6193\n",
      "Processing molecule 6000/6193\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/6955\n",
      "Processing molecule 2000/6955\n",
      "Processing molecule 3000/6955\n",
      "Processing molecule 4000/6955\n",
      "Processing molecule 5000/6955\n",
      "Processing molecule 6000/6955\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/6450\n",
      "Processing molecule 2000/6450\n",
      "Processing molecule 3000/6450\n",
      "Processing molecule 4000/6450\n",
      "Processing molecule 5000/6450\n",
      "Processing molecule 6000/6450\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/5832\n",
      "Processing molecule 2000/5832\n",
      "Processing molecule 3000/5832\n",
      "Processing molecule 4000/5832\n",
      "Processing molecule 5000/5832\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/7072\n",
      "Processing molecule 2000/7072\n",
      "Processing molecule 3000/7072\n",
      "Processing molecule 4000/7072\n",
      "Processing molecule 5000/7072\n",
      "Processing molecule 6000/7072\n",
      "Processing molecule 7000/7072\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/6467\n",
      "Processing molecule 2000/6467\n",
      "Processing molecule 3000/6467\n",
      "Processing molecule 4000/6467\n",
      "Processing molecule 5000/6467\n",
      "Processing molecule 6000/6467\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/5810\n",
      "Processing molecule 2000/5810\n",
      "Processing molecule 3000/5810\n",
      "Processing molecule 4000/5810\n",
      "Processing molecule 5000/5810\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/6774\n",
      "Processing molecule 2000/6774\n",
      "Processing molecule 3000/6774\n",
      "Processing molecule 4000/6774\n",
      "Processing molecule 5000/6774\n",
      "Processing molecule 6000/6774\n",
      "Data created!!\n"
     ]
    }
   ],
   "source": [
    "data_ds = []\n",
    "for i, task in  enumerate(tox21_tasks):\n",
    "    a = df[['smiles' , task]]\n",
    "    a = a.dropna()\n",
    "    ds = DATASET(a, smiles_to_bigraph, AttentiveFPAtomFeaturizer(), cache_file_path = cache_path)\n",
    "    data = create_dataset_with_gcn(ds, embed_class_tox21, gcn_model, tox21_tasks, i)\n",
    "    for d in data:\n",
    "        data_ds.append(d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "hidden": true,
    "id": "9wZRIKrq2Kec",
    "outputId": "6a5e45aa-32cb-47da-d25c-325f6cfe106b",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train positive label: 5276 - train negative label: 64875\n",
      "up and down sampling => train positive label: 36932 - train negative label: 64875\n",
      "Test positive label: 586 - Test negative label: 7209\n",
      "Epoch 1/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.5229 - accuracy: 0.7523 - mae: 0.3434 - mse: 0.1711 - auc: 0.8021\n",
      "Epoch 2/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.4708 - accuracy: 0.7833 - mae: 0.3049 - mse: 0.1514 - auc: 0.8434\n",
      "Epoch 3/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.4562 - accuracy: 0.7923 - mae: 0.2938 - mse: 0.1458 - auc: 0.8537\n",
      "Epoch 4/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.4442 - accuracy: 0.7983 - mae: 0.2859 - mse: 0.1419 - auc: 0.8615\n",
      "Epoch 5/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.4355 - accuracy: 0.8046 - mae: 0.2793 - mse: 0.1387 - auc: 0.8674\n",
      "Epoch 6/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.4256 - accuracy: 0.8089 - mae: 0.2723 - mse: 0.1352 - auc: 0.8738\n",
      "Epoch 7/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.4177 - accuracy: 0.8140 - mae: 0.2666 - mse: 0.1323 - auc: 0.8789\n",
      "Epoch 8/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.4124 - accuracy: 0.8153 - mae: 0.2634 - mse: 0.1309 - auc: 0.8821\n",
      "Epoch 9/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.4055 - accuracy: 0.8181 - mae: 0.2583 - mse: 0.1287 - auc: 0.8866\n",
      "Epoch 10/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.4010 - accuracy: 0.8209 - mae: 0.2554 - mse: 0.1268 - auc: 0.8894\n",
      "81\n",
      "81\n",
      "81\n",
      "80\n",
      "80\n",
      "79\n",
      "79\n",
      "78\n",
      "75\n",
      "Epoch 1/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.3932 - accuracy: 0.8262 - mae: 0.2499 - mse: 0.1239 - auc: 0.8942\n",
      "Epoch 2/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.3891 - accuracy: 0.8280 - mae: 0.2467 - mse: 0.1227 - auc: 0.8964\n",
      "Epoch 3/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.3828 - accuracy: 0.8302 - mae: 0.2426 - mse: 0.1206 - auc: 0.8999\n",
      "Epoch 4/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.3769 - accuracy: 0.8338 - mae: 0.2384 - mse: 0.1187 - auc: 0.9030\n",
      "Epoch 5/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.3710 - accuracy: 0.8368 - mae: 0.2347 - mse: 0.1166 - auc: 0.9063\n",
      "Epoch 6/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.3660 - accuracy: 0.8393 - mae: 0.2310 - mse: 0.1149 - auc: 0.9091\n",
      "Epoch 7/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.3614 - accuracy: 0.8420 - mae: 0.2280 - mse: 0.1134 - auc: 0.9115\n",
      "Epoch 8/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.3588 - accuracy: 0.8440 - mae: 0.2256 - mse: 0.1123 - auc: 0.9131\n",
      "Epoch 9/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.3521 - accuracy: 0.8474 - mae: 0.2210 - mse: 0.1100 - auc: 0.9163\n",
      "Epoch 10/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.3489 - accuracy: 0.8477 - mae: 0.2196 - mse: 0.1091 - auc: 0.9180\n",
      "84\n",
      "84\n",
      "84\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "82\n",
      "82\n",
      "Epoch 1/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.3419 - accuracy: 0.8507 - mae: 0.2147 - mse: 0.1070 - auc: 0.9214\n",
      "Epoch 2/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.3391 - accuracy: 0.8518 - mae: 0.2131 - mse: 0.1060 - auc: 0.9229\n",
      "Epoch 3/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.3363 - accuracy: 0.8543 - mae: 0.2110 - mse: 0.1049 - auc: 0.9242\n",
      "Epoch 4/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.3313 - accuracy: 0.8564 - mae: 0.2077 - mse: 0.1033 - auc: 0.9265\n",
      "Epoch 5/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.3276 - accuracy: 0.8596 - mae: 0.2049 - mse: 0.1020 - auc: 0.9281\n",
      "Epoch 6/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.3201 - accuracy: 0.8619 - mae: 0.2001 - mse: 0.0997 - auc: 0.9315\n",
      "Epoch 7/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.3206 - accuracy: 0.8616 - mae: 0.1999 - mse: 0.0997 - auc: 0.9314\n",
      "Epoch 8/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.3203 - accuracy: 0.8624 - mae: 0.2001 - mse: 0.0997 - auc: 0.9315\n",
      "Epoch 9/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.3154 - accuracy: 0.8655 - mae: 0.1964 - mse: 0.0978 - auc: 0.9336\n",
      "Epoch 10/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.3128 - accuracy: 0.8654 - mae: 0.1950 - mse: 0.0971 - auc: 0.9348\n",
      "86\n",
      "86\n",
      "86\n",
      "86\n",
      "85\n",
      "85\n",
      "85\n",
      "85\n",
      "85\n",
      "Epoch 1/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.3077 - accuracy: 0.8676 - mae: 0.1914 - mse: 0.0952 - auc: 0.9371\n",
      "Epoch 2/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.3051 - accuracy: 0.8690 - mae: 0.1899 - mse: 0.0945 - auc: 0.9382\n",
      "Epoch 3/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.3001 - accuracy: 0.8714 - mae: 0.1864 - mse: 0.0928 - auc: 0.9402\n",
      "Epoch 4/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.3000 - accuracy: 0.8715 - mae: 0.1867 - mse: 0.0928 - auc: 0.9401\n",
      "Epoch 5/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2995 - accuracy: 0.8722 - mae: 0.1857 - mse: 0.0924 - auc: 0.9404\n",
      "Epoch 6/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2943 - accuracy: 0.8735 - mae: 0.1827 - mse: 0.0911 - auc: 0.9423\n",
      "Epoch 7/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2914 - accuracy: 0.8746 - mae: 0.1807 - mse: 0.0901 - auc: 0.9434\n",
      "Epoch 8/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2914 - accuracy: 0.8758 - mae: 0.1805 - mse: 0.0900 - auc: 0.9436\n",
      "Epoch 9/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2902 - accuracy: 0.8752 - mae: 0.1801 - mse: 0.0898 - auc: 0.9440\n",
      "Epoch 10/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2908 - accuracy: 0.8762 - mae: 0.1797 - mse: 0.0895 - auc: 0.9438\n",
      "87\n",
      "87\n",
      "87\n",
      "87\n",
      "87\n",
      "87\n",
      "87\n",
      "86\n",
      "86\n",
      "Epoch 1/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2837 - accuracy: 0.8790 - mae: 0.1756 - mse: 0.0874 - auc: 0.9465\n",
      "Epoch 2/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2836 - accuracy: 0.8779 - mae: 0.1754 - mse: 0.0875 - auc: 0.9465\n",
      "Epoch 3/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2831 - accuracy: 0.8785 - mae: 0.1750 - mse: 0.0872 - auc: 0.9467\n",
      "Epoch 4/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2778 - accuracy: 0.8816 - mae: 0.1718 - mse: 0.0854 - auc: 0.9489\n",
      "Epoch 5/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2786 - accuracy: 0.8819 - mae: 0.1716 - mse: 0.0855 - auc: 0.9484\n",
      "Epoch 6/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2754 - accuracy: 0.8821 - mae: 0.1703 - mse: 0.0849 - auc: 0.9495\n",
      "Epoch 7/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2756 - accuracy: 0.8830 - mae: 0.1693 - mse: 0.0844 - auc: 0.9495\n",
      "Epoch 8/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2738 - accuracy: 0.8838 - mae: 0.1692 - mse: 0.0842 - auc: 0.9502\n",
      "Epoch 9/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2703 - accuracy: 0.8857 - mae: 0.1662 - mse: 0.0829 - auc: 0.9514\n",
      "Epoch 10/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2696 - accuracy: 0.8863 - mae: 0.1652 - mse: 0.0824 - auc: 0.9517\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "87\n",
      "87\n",
      "87\n",
      "Epoch 1/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2673 - accuracy: 0.8866 - mae: 0.1643 - mse: 0.0819 - auc: 0.9523\n",
      "Epoch 2/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2670 - accuracy: 0.8869 - mae: 0.1644 - mse: 0.0818 - auc: 0.9526\n",
      "Epoch 3/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2671 - accuracy: 0.8869 - mae: 0.1642 - mse: 0.0819 - auc: 0.9525\n",
      "Epoch 4/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2680 - accuracy: 0.8870 - mae: 0.1646 - mse: 0.0821 - auc: 0.9522\n",
      "Epoch 5/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2653 - accuracy: 0.8882 - mae: 0.1628 - mse: 0.0811 - auc: 0.9532\n",
      "Epoch 6/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2655 - accuracy: 0.8875 - mae: 0.1637 - mse: 0.0815 - auc: 0.9530\n",
      "Epoch 7/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2623 - accuracy: 0.8893 - mae: 0.1612 - mse: 0.0802 - auc: 0.9542\n",
      "Epoch 8/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2630 - accuracy: 0.8884 - mae: 0.1613 - mse: 0.0808 - auc: 0.9538\n",
      "Epoch 9/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2639 - accuracy: 0.8888 - mae: 0.1616 - mse: 0.0806 - auc: 0.9535\n",
      "Epoch 10/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2589 - accuracy: 0.8907 - mae: 0.1584 - mse: 0.0791 - auc: 0.9553\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "Epoch 1/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2571 - accuracy: 0.8924 - mae: 0.1574 - mse: 0.0784 - auc: 0.9558\n",
      "Epoch 2/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2562 - accuracy: 0.8920 - mae: 0.1570 - mse: 0.0784 - auc: 0.9561\n",
      "Epoch 3/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2541 - accuracy: 0.8928 - mae: 0.1554 - mse: 0.0775 - auc: 0.9569\n",
      "Epoch 4/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2536 - accuracy: 0.8939 - mae: 0.1549 - mse: 0.0772 - auc: 0.9571\n",
      "Epoch 5/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2548 - accuracy: 0.8919 - mae: 0.1568 - mse: 0.0780 - auc: 0.9567\n",
      "Epoch 6/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2597 - accuracy: 0.8900 - mae: 0.1596 - mse: 0.0796 - auc: 0.9550\n",
      "Epoch 7/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2580 - accuracy: 0.8912 - mae: 0.1578 - mse: 0.0790 - auc: 0.9556\n",
      "Epoch 8/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2537 - accuracy: 0.8934 - mae: 0.1551 - mse: 0.0773 - auc: 0.9571\n",
      "Epoch 9/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2531 - accuracy: 0.8940 - mae: 0.1549 - mse: 0.0774 - auc: 0.9573\n",
      "Epoch 10/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2538 - accuracy: 0.8934 - mae: 0.1556 - mse: 0.0774 - auc: 0.9571\n",
      "89\n",
      "89\n",
      "89\n",
      "88\n",
      "89\n",
      "89\n",
      "89\n",
      "89\n",
      "89\n",
      "Epoch 1/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2540 - accuracy: 0.8921 - mae: 0.1561 - mse: 0.0780 - auc: 0.9570\n",
      "Epoch 2/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2536 - accuracy: 0.8939 - mae: 0.1554 - mse: 0.0774 - auc: 0.9570\n",
      "Epoch 3/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2502 - accuracy: 0.8950 - mae: 0.1528 - mse: 0.0763 - auc: 0.9581\n",
      "Epoch 4/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2494 - accuracy: 0.8944 - mae: 0.1527 - mse: 0.0763 - auc: 0.9584\n",
      "Epoch 5/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2507 - accuracy: 0.8947 - mae: 0.1537 - mse: 0.0764 - auc: 0.9580\n",
      "Epoch 6/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2524 - accuracy: 0.8943 - mae: 0.1540 - mse: 0.0768 - auc: 0.9575\n",
      "Epoch 7/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2498 - accuracy: 0.8947 - mae: 0.1531 - mse: 0.0763 - auc: 0.9583\n",
      "Epoch 8/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2516 - accuracy: 0.8946 - mae: 0.1543 - mse: 0.0767 - auc: 0.9578\n",
      "Epoch 9/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2487 - accuracy: 0.8950 - mae: 0.1524 - mse: 0.0760 - auc: 0.9587\n",
      "Epoch 10/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2454 - accuracy: 0.8957 - mae: 0.1501 - mse: 0.0749 - auc: 0.9596\n",
      "89\n",
      "89\n",
      "89\n",
      "89\n",
      "89\n",
      "89\n",
      "89\n",
      "89\n",
      "89\n",
      "8\n",
      "244/244 [==============================] - 0s 673us/step - loss: 0.2386 - accuracy: 0.8980 - mae: 0.1405 - mse: 0.0707 - auc: 0.8962\n",
      "Save_model\n",
      "train positive label: 5232 - train negative label: 64919\n",
      "up and down sampling => train positive label: 36624 - train negative label: 64919\n",
      "Test positive label: 630 - Test negative label: 7165\n",
      "Epoch 1/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.5220 - accuracy: 0.7532 - mae: 0.3434 - mse: 0.1712 - auc_1: 0.8016\n",
      "Epoch 2/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.4694 - accuracy: 0.7884 - mae: 0.3041 - mse: 0.1507 - auc_1: 0.8432\n",
      "Epoch 3/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.4545 - accuracy: 0.7962 - mae: 0.2932 - mse: 0.1453 - auc_1: 0.8538\n",
      "Epoch 4/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.4434 - accuracy: 0.8006 - mae: 0.2850 - mse: 0.1414 - auc_1: 0.8617\n",
      "Epoch 5/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.4357 - accuracy: 0.8046 - mae: 0.2795 - mse: 0.1388 - auc_1: 0.8666\n",
      "Epoch 6/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.4263 - accuracy: 0.8095 - mae: 0.2729 - mse: 0.1355 - auc_1: 0.8733\n",
      "Epoch 7/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.4189 - accuracy: 0.8134 - mae: 0.2679 - mse: 0.1328 - auc_1: 0.8780\n",
      "Epoch 8/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.4116 - accuracy: 0.8161 - mae: 0.2632 - mse: 0.1306 - auc_1: 0.8825\n",
      "Epoch 9/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.4039 - accuracy: 0.8187 - mae: 0.2577 - mse: 0.1281 - auc_1: 0.8872\n",
      "Epoch 10/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.3980 - accuracy: 0.8235 - mae: 0.2536 - mse: 0.1258 - auc_1: 0.8910\n",
      "81\n",
      "81\n",
      "81\n",
      "80\n",
      "80\n",
      "80\n",
      "79\n",
      "78\n",
      "75\n",
      "Epoch 1/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.3913 - accuracy: 0.8260 - mae: 0.2492 - mse: 0.1237 - auc_1: 0.8948\n",
      "Epoch 2/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.3853 - accuracy: 0.8288 - mae: 0.2453 - mse: 0.1217 - auc_1: 0.8986\n",
      "Epoch 3/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.3801 - accuracy: 0.8298 - mae: 0.2416 - mse: 0.1200 - auc_1: 0.9016\n",
      "Epoch 4/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.3739 - accuracy: 0.8351 - mae: 0.2376 - mse: 0.1181 - auc_1: 0.9045\n",
      "Epoch 5/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.3688 - accuracy: 0.8368 - mae: 0.2341 - mse: 0.1162 - auc_1: 0.9076\n",
      "Epoch 6/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.3635 - accuracy: 0.8393 - mae: 0.2305 - mse: 0.1144 - auc_1: 0.9105\n",
      "Epoch 7/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.3564 - accuracy: 0.8427 - mae: 0.2255 - mse: 0.1121 - auc_1: 0.9141\n",
      "Epoch 8/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.3520 - accuracy: 0.8442 - mae: 0.2231 - mse: 0.1110 - auc_1: 0.9160\n",
      "Epoch 9/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.3475 - accuracy: 0.8470 - mae: 0.2198 - mse: 0.1092 - auc_1: 0.9184\n",
      "Epoch 10/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.3429 - accuracy: 0.8479 - mae: 0.2168 - mse: 0.1078 - auc_1: 0.9208\n",
      "84\n",
      "84\n",
      "84\n",
      "83\n",
      "83\n",
      "83\n",
      "82\n",
      "82\n",
      "82\n",
      "Epoch 1/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.3375 - accuracy: 0.8510 - mae: 0.2132 - mse: 0.1060 - auc_1: 0.9234\n",
      "Epoch 2/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.3352 - accuracy: 0.8520 - mae: 0.2119 - mse: 0.1053 - auc_1: 0.9245\n",
      "Epoch 3/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.3285 - accuracy: 0.8552 - mae: 0.2071 - mse: 0.1030 - auc_1: 0.9277\n",
      "Epoch 4/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.3249 - accuracy: 0.8558 - mae: 0.2045 - mse: 0.1019 - auc_1: 0.9292\n",
      "Epoch 5/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.3223 - accuracy: 0.8574 - mae: 0.2033 - mse: 0.1010 - auc_1: 0.9304\n",
      "Epoch 6/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.3192 - accuracy: 0.8581 - mae: 0.2015 - mse: 0.1002 - auc_1: 0.9317\n",
      "Epoch 7/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.3151 - accuracy: 0.8611 - mae: 0.1983 - mse: 0.0985 - auc_1: 0.9336\n",
      "Epoch 8/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.3125 - accuracy: 0.8616 - mae: 0.1961 - mse: 0.0976 - auc_1: 0.9347\n",
      "Epoch 9/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.3082 - accuracy: 0.8636 - mae: 0.1937 - mse: 0.0964 - auc_1: 0.9365\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 10/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.3067 - accuracy: 0.8649 - mae: 0.1928 - mse: 0.0959 - auc_1: 0.9372\n",
      "86\n",
      "86\n",
      "86\n",
      "85\n",
      "85\n",
      "85\n",
      "85\n",
      "85\n",
      "85\n",
      "Epoch 1/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.3020 - accuracy: 0.8679 - mae: 0.1892 - mse: 0.0941 - auc_1: 0.9392\n",
      "Epoch 2/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.3002 - accuracy: 0.8686 - mae: 0.1881 - mse: 0.0936 - auc_1: 0.9398\n",
      "Epoch 3/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2957 - accuracy: 0.8715 - mae: 0.1850 - mse: 0.0918 - auc_1: 0.9418\n",
      "Epoch 4/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2947 - accuracy: 0.8711 - mae: 0.1841 - mse: 0.0917 - auc_1: 0.9420\n",
      "Epoch 5/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2914 - accuracy: 0.8739 - mae: 0.1817 - mse: 0.0904 - auc_1: 0.9434\n",
      "Epoch 6/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2901 - accuracy: 0.8745 - mae: 0.1806 - mse: 0.0900 - auc_1: 0.9439\n",
      "Epoch 7/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2884 - accuracy: 0.8754 - mae: 0.1797 - mse: 0.0892 - auc_1: 0.9446\n",
      "Epoch 8/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2894 - accuracy: 0.8744 - mae: 0.1806 - mse: 0.0898 - auc_1: 0.9441\n",
      "Epoch 9/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2876 - accuracy: 0.8755 - mae: 0.1788 - mse: 0.0891 - auc_1: 0.9449\n",
      "Epoch 10/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2837 - accuracy: 0.8790 - mae: 0.1763 - mse: 0.0876 - auc_1: 0.9464\n",
      "87\n",
      "87\n",
      "87\n",
      "87\n",
      "87\n",
      "87\n",
      "87\n",
      "86\n",
      "86\n",
      "Epoch 1/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2815 - accuracy: 0.8794 - mae: 0.1744 - mse: 0.0870 - auc_1: 0.9471\n",
      "Epoch 2/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2757 - accuracy: 0.8807 - mae: 0.1713 - mse: 0.0854 - auc_1: 0.9491\n",
      "Epoch 3/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2767 - accuracy: 0.8808 - mae: 0.1719 - mse: 0.0855 - auc_1: 0.9490\n",
      "Epoch 4/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2726 - accuracy: 0.8832 - mae: 0.1691 - mse: 0.0843 - auc_1: 0.9504\n",
      "Epoch 5/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2741 - accuracy: 0.8816 - mae: 0.1702 - mse: 0.0847 - auc_1: 0.9498\n",
      "Epoch 6/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2733 - accuracy: 0.8816 - mae: 0.1696 - mse: 0.0844 - auc_1: 0.9503\n",
      "Epoch 7/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2709 - accuracy: 0.8837 - mae: 0.1681 - mse: 0.0835 - auc_1: 0.9511\n",
      "Epoch 8/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2701 - accuracy: 0.8850 - mae: 0.1671 - mse: 0.0833 - auc_1: 0.9512\n",
      "Epoch 9/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2685 - accuracy: 0.8851 - mae: 0.1663 - mse: 0.0829 - auc_1: 0.9518\n",
      "Epoch 10/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2687 - accuracy: 0.8857 - mae: 0.1662 - mse: 0.0828 - auc_1: 0.9517\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "87\n",
      "Epoch 1/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2658 - accuracy: 0.8860 - mae: 0.1651 - mse: 0.0820 - auc_1: 0.9528\n",
      "Epoch 2/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2630 - accuracy: 0.8872 - mae: 0.1626 - mse: 0.0811 - auc_1: 0.9537\n",
      "Epoch 3/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2647 - accuracy: 0.8873 - mae: 0.1637 - mse: 0.0815 - auc_1: 0.9530\n",
      "Epoch 4/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2665 - accuracy: 0.8857 - mae: 0.1646 - mse: 0.0820 - auc_1: 0.9527\n",
      "Epoch 5/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2631 - accuracy: 0.8873 - mae: 0.1627 - mse: 0.0811 - auc_1: 0.9537\n",
      "Epoch 6/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2644 - accuracy: 0.8867 - mae: 0.1640 - mse: 0.0815 - auc_1: 0.9531\n",
      "Epoch 7/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2609 - accuracy: 0.8891 - mae: 0.1612 - mse: 0.0803 - auc_1: 0.9544\n",
      "Epoch 8/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2599 - accuracy: 0.8898 - mae: 0.1605 - mse: 0.0799 - auc_1: 0.9548\n",
      "Epoch 9/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2628 - accuracy: 0.8881 - mae: 0.1622 - mse: 0.0809 - auc_1: 0.9536\n",
      "Epoch 10/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2584 - accuracy: 0.8906 - mae: 0.1594 - mse: 0.0794 - auc_1: 0.9552\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "Epoch 1/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2614 - accuracy: 0.8893 - mae: 0.1608 - mse: 0.0803 - auc_1: 0.9543\n",
      "Epoch 2/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2658 - accuracy: 0.8869 - mae: 0.1640 - mse: 0.0819 - auc_1: 0.9528\n",
      "Epoch 3/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2632 - accuracy: 0.8870 - mae: 0.1629 - mse: 0.0811 - auc_1: 0.9537\n",
      "Epoch 4/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2605 - accuracy: 0.8901 - mae: 0.1608 - mse: 0.0801 - auc_1: 0.9544\n",
      "Epoch 5/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2597 - accuracy: 0.8904 - mae: 0.1602 - mse: 0.0797 - auc_1: 0.9548\n",
      "Epoch 6/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2587 - accuracy: 0.8896 - mae: 0.1601 - mse: 0.0797 - auc_1: 0.9552\n",
      "Epoch 7/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2570 - accuracy: 0.8907 - mae: 0.1590 - mse: 0.0791 - auc_1: 0.9556\n",
      "Epoch 8/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2576 - accuracy: 0.8901 - mae: 0.1593 - mse: 0.0795 - auc_1: 0.9555\n",
      "Epoch 9/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2577 - accuracy: 0.8906 - mae: 0.1590 - mse: 0.0794 - auc_1: 0.9555\n",
      "Epoch 10/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2560 - accuracy: 0.8912 - mae: 0.1577 - mse: 0.0787 - auc_1: 0.9560\n",
      "89\n",
      "89\n",
      "89\n",
      "88\n",
      "89\n",
      "89\n",
      "88\n",
      "88\n",
      "88\n",
      "Epoch 1/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2559 - accuracy: 0.8917 - mae: 0.1580 - mse: 0.0787 - auc_1: 0.9561\n",
      "Epoch 2/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2617 - accuracy: 0.8882 - mae: 0.1621 - mse: 0.0806 - auc_1: 0.9542\n",
      "Epoch 3/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2578 - accuracy: 0.8901 - mae: 0.1594 - mse: 0.0794 - auc_1: 0.9555\n",
      "Epoch 4/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2553 - accuracy: 0.8924 - mae: 0.1577 - mse: 0.0782 - auc_1: 0.9563\n",
      "Epoch 5/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2568 - accuracy: 0.8896 - mae: 0.1587 - mse: 0.0793 - auc_1: 0.9558\n",
      "Epoch 6/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2528 - accuracy: 0.8923 - mae: 0.1562 - mse: 0.0779 - auc_1: 0.9570\n",
      "Epoch 7/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2502 - accuracy: 0.8933 - mae: 0.1544 - mse: 0.0769 - auc_1: 0.9579\n",
      "Epoch 8/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2503 - accuracy: 0.8939 - mae: 0.1542 - mse: 0.0767 - auc_1: 0.9581\n",
      "Epoch 9/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2496 - accuracy: 0.8951 - mae: 0.1533 - mse: 0.0763 - auc_1: 0.9581\n",
      "Epoch 10/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2451 - accuracy: 0.8976 - mae: 0.1506 - mse: 0.0749 - auc_1: 0.9596\n",
      "89\n",
      "89\n",
      "89\n",
      "89\n",
      "88\n",
      "89\n",
      "89\n",
      "88\n",
      "89\n",
      "Epoch 1/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2454 - accuracy: 0.8966 - mae: 0.1506 - mse: 0.0753 - auc_1: 0.9595\n",
      "Epoch 2/10\n",
      "794/794 [==============================] - 4s 6ms/step - loss: 0.2467 - accuracy: 0.8957 - mae: 0.1515 - mse: 0.0755 - auc_1: 0.9590\n",
      "Epoch 3/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2448 - accuracy: 0.8970 - mae: 0.1502 - mse: 0.0749 - auc_1: 0.9597\n",
      "Epoch 4/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2495 - accuracy: 0.8956 - mae: 0.1533 - mse: 0.0763 - auc_1: 0.9581\n",
      "Epoch 5/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2484 - accuracy: 0.8946 - mae: 0.1532 - mse: 0.0763 - auc_1: 0.9587\n",
      "Epoch 6/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2402 - accuracy: 0.8995 - mae: 0.1472 - mse: 0.0733 - auc_1: 0.9612\n",
      "Epoch 7/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2456 - accuracy: 0.8969 - mae: 0.1509 - mse: 0.0750 - auc_1: 0.9596\n",
      "Epoch 8/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2417 - accuracy: 0.8976 - mae: 0.1484 - mse: 0.0739 - auc_1: 0.9607\n",
      "Epoch 9/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2402 - accuracy: 0.9003 - mae: 0.1473 - mse: 0.0732 - auc_1: 0.9613\n",
      "Epoch 10/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2392 - accuracy: 0.9001 - mae: 0.1466 - mse: 0.0731 - auc_1: 0.9614\n",
      "90\n",
      "89\n",
      "89\n",
      "89\n",
      "89\n",
      "89\n",
      "89\n",
      "89\n",
      "89\n",
      "Epoch 1/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2379 - accuracy: 0.9020 - mae: 0.1455 - mse: 0.0725 - auc_1: 0.9619\n",
      "Epoch 2/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2371 - accuracy: 0.9009 - mae: 0.1453 - mse: 0.0722 - auc_1: 0.9622\n",
      "Epoch 3/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2416 - accuracy: 0.8988 - mae: 0.1484 - mse: 0.0739 - auc_1: 0.9607\n",
      "Epoch 4/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2398 - accuracy: 0.8986 - mae: 0.1480 - mse: 0.0737 - auc_1: 0.9612\n",
      "Epoch 5/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2437 - accuracy: 0.8972 - mae: 0.1500 - mse: 0.0748 - auc_1: 0.9600\n",
      "Epoch 6/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2412 - accuracy: 0.8984 - mae: 0.1482 - mse: 0.0738 - auc_1: 0.9609\n",
      "Epoch 7/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2392 - accuracy: 0.8993 - mae: 0.1472 - mse: 0.0734 - auc_1: 0.9615\n",
      "Epoch 8/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2406 - accuracy: 0.8992 - mae: 0.1474 - mse: 0.0734 - auc_1: 0.9610\n",
      "Epoch 9/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2380 - accuracy: 0.9002 - mae: 0.1459 - mse: 0.0728 - auc_1: 0.9619\n",
      "Epoch 10/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2377 - accuracy: 0.9004 - mae: 0.1464 - mse: 0.0728 - auc_1: 0.9619\n",
      "90\n",
      "89\n",
      "89\n",
      "89\n",
      "89\n",
      "89\n",
      "89\n",
      "90\n",
      "90\n",
      "Epoch 1/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2381 - accuracy: 0.8994 - mae: 0.1462 - mse: 0.0731 - auc_1: 0.9617\n",
      "Epoch 2/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2365 - accuracy: 0.9004 - mae: 0.1458 - mse: 0.0725 - auc_1: 0.9622\n",
      "Epoch 3/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2337 - accuracy: 0.9029 - mae: 0.1432 - mse: 0.0713 - auc_1: 0.9631\n",
      "Epoch 4/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2389 - accuracy: 0.9001 - mae: 0.1465 - mse: 0.0729 - auc_1: 0.9616\n",
      "Epoch 5/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2389 - accuracy: 0.8996 - mae: 0.1466 - mse: 0.0731 - auc_1: 0.9616\n",
      "Epoch 6/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2360 - accuracy: 0.9017 - mae: 0.1444 - mse: 0.0719 - auc_1: 0.9625\n",
      "Epoch 7/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2351 - accuracy: 0.9005 - mae: 0.1446 - mse: 0.0721 - auc_1: 0.9627\n",
      "Epoch 8/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2373 - accuracy: 0.9008 - mae: 0.1460 - mse: 0.0727 - auc_1: 0.9620\n",
      "Epoch 9/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2390 - accuracy: 0.9003 - mae: 0.1465 - mse: 0.0729 - auc_1: 0.9616\n",
      "Epoch 10/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2377 - accuracy: 0.9002 - mae: 0.1459 - mse: 0.0727 - auc_1: 0.9620\n",
      "90\n",
      "90\n",
      "90\n",
      "90\n",
      "89\n",
      "90\n",
      "90\n",
      "90\n",
      "89\n",
      "Epoch 1/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2366 - accuracy: 0.9012 - mae: 0.1451 - mse: 0.0724 - auc_1: 0.9621\n",
      "Epoch 2/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2343 - accuracy: 0.9023 - mae: 0.1439 - mse: 0.0716 - auc_1: 0.9629\n",
      "Epoch 3/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2339 - accuracy: 0.9019 - mae: 0.1433 - mse: 0.0714 - auc_1: 0.9632\n",
      "Epoch 4/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2350 - accuracy: 0.9020 - mae: 0.1444 - mse: 0.0718 - auc_1: 0.9626\n",
      "Epoch 5/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2351 - accuracy: 0.9024 - mae: 0.1439 - mse: 0.0717 - auc_1: 0.9625\n",
      "Epoch 6/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2342 - accuracy: 0.9014 - mae: 0.1439 - mse: 0.0718 - auc_1: 0.9629\n",
      "Epoch 7/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2333 - accuracy: 0.9018 - mae: 0.1434 - mse: 0.0714 - auc_1: 0.9631\n",
      "Epoch 8/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2330 - accuracy: 0.9018 - mae: 0.1433 - mse: 0.0713 - auc_1: 0.9635\n",
      "Epoch 9/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2327 - accuracy: 0.9017 - mae: 0.1434 - mse: 0.0714 - auc_1: 0.9633\n",
      "Epoch 10/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2312 - accuracy: 0.9032 - mae: 0.1419 - mse: 0.0706 - auc_1: 0.9638\n",
      "90\n",
      "90\n",
      "90\n",
      "90\n",
      "90\n",
      "90\n",
      "90\n",
      "90\n",
      "90\n",
      "12\n",
      "  1/244 [..............................] - ETA: 0s - loss: 0.1576 - accuracy: 0.9062 - mae: 0.0995 - mse: 0.0524 - auc_1: 0.0000e+00WARNING:tensorflow:Callbacks method `on_test_batch_end` is slow compared to the batch time (batch time: 0.0000s vs `on_test_batch_end` time: 0.0010s). Check your callbacks.\n",
      "244/244 [==============================] - 0s 710us/step - loss: 0.2476 - accuracy: 0.9028 - mae: 0.1233 - mse: 0.0689 - auc_1: 0.8820\n",
      "train positive label: 5247 - train negative label: 64904\n",
      "up and down sampling => train positive label: 36729 - train negative label: 64904\n",
      "Test positive label: 615 - Test negative label: 7180\n",
      "Epoch 1/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.5277 - accuracy: 0.7508 - mae: 0.3467 - mse: 0.1728 - auc_2: 0.7975\n",
      "Epoch 2/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.4757 - accuracy: 0.7848 - mae: 0.3091 - mse: 0.1529 - auc_2: 0.8393\n",
      "Epoch 3/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.4624 - accuracy: 0.7923 - mae: 0.2991 - mse: 0.1479 - auc_2: 0.8488\n",
      "Epoch 4/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.4516 - accuracy: 0.7956 - mae: 0.2918 - mse: 0.1445 - auc_2: 0.8560\n",
      "Epoch 5/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.4386 - accuracy: 0.8021 - mae: 0.2825 - mse: 0.1400 - auc_2: 0.8650\n",
      "Epoch 6/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.4325 - accuracy: 0.8060 - mae: 0.2777 - mse: 0.1377 - auc_2: 0.8692\n",
      "Epoch 7/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.4240 - accuracy: 0.8096 - mae: 0.2718 - mse: 0.1350 - auc_2: 0.8746\n",
      "Epoch 8/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.4173 - accuracy: 0.8121 - mae: 0.2671 - mse: 0.1328 - auc_2: 0.8789\n",
      "Epoch 9/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.4109 - accuracy: 0.8151 - mae: 0.2625 - mse: 0.1306 - auc_2: 0.8830\n",
      "Epoch 10/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.4078 - accuracy: 0.8166 - mae: 0.2602 - mse: 0.1294 - auc_2: 0.8853\n",
      "81\n",
      "81\n",
      "80\n",
      "80\n",
      "80\n",
      "79\n",
      "79\n",
      "78\n",
      "75\n",
      "Epoch 1/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.3997 - accuracy: 0.8210 - mae: 0.2549 - mse: 0.1267 - auc_2: 0.8899\n",
      "Epoch 2/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.3965 - accuracy: 0.8219 - mae: 0.2530 - mse: 0.1257 - auc_2: 0.8920\n",
      "Epoch 3/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.3879 - accuracy: 0.8256 - mae: 0.2470 - mse: 0.1230 - auc_2: 0.8969\n",
      "Epoch 4/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.3832 - accuracy: 0.8279 - mae: 0.2444 - mse: 0.1213 - auc_2: 0.8997\n",
      "Epoch 5/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.3780 - accuracy: 0.8318 - mae: 0.2402 - mse: 0.1196 - auc_2: 0.9027\n",
      "Epoch 6/10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "795/795 [==============================] - 4s 5ms/step - loss: 0.3742 - accuracy: 0.8324 - mae: 0.2379 - mse: 0.1183 - auc_2: 0.9047\n",
      "Epoch 7/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.3683 - accuracy: 0.8363 - mae: 0.2337 - mse: 0.1162 - auc_2: 0.9078\n",
      "Epoch 8/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.3614 - accuracy: 0.8387 - mae: 0.2297 - mse: 0.1143 - auc_2: 0.9114\n",
      "Epoch 9/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.3602 - accuracy: 0.8393 - mae: 0.2285 - mse: 0.1137 - auc_2: 0.9122\n",
      "Epoch 10/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.3556 - accuracy: 0.8415 - mae: 0.2251 - mse: 0.1121 - auc_2: 0.9145\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "82\n",
      "82\n",
      "82\n",
      "82\n",
      "Epoch 1/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.3509 - accuracy: 0.8441 - mae: 0.2216 - mse: 0.1107 - auc_2: 0.9168\n",
      "Epoch 2/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.3481 - accuracy: 0.8468 - mae: 0.2200 - mse: 0.1095 - auc_2: 0.9184\n",
      "Epoch 3/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.3428 - accuracy: 0.8486 - mae: 0.2168 - mse: 0.1080 - auc_2: 0.9207\n",
      "Epoch 4/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.3394 - accuracy: 0.8501 - mae: 0.2142 - mse: 0.1067 - auc_2: 0.9225\n",
      "Epoch 5/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.3407 - accuracy: 0.8498 - mae: 0.2144 - mse: 0.1070 - auc_2: 0.9220\n",
      "Epoch 6/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.3334 - accuracy: 0.8530 - mae: 0.2102 - mse: 0.1048 - auc_2: 0.9253\n",
      "Epoch 7/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.3298 - accuracy: 0.8548 - mae: 0.2083 - mse: 0.1036 - auc_2: 0.9270\n",
      "Epoch 8/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.3267 - accuracy: 0.8564 - mae: 0.2055 - mse: 0.1026 - auc_2: 0.9285\n",
      "Epoch 9/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.3235 - accuracy: 0.8573 - mae: 0.2037 - mse: 0.1016 - auc_2: 0.9299\n",
      "Epoch 10/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.3198 - accuracy: 0.8594 - mae: 0.2020 - mse: 0.1003 - auc_2: 0.9315\n",
      "85\n",
      "85\n",
      "85\n",
      "85\n",
      "84\n",
      "85\n",
      "84\n",
      "84\n",
      "84\n",
      "Epoch 1/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.3162 - accuracy: 0.8622 - mae: 0.1984 - mse: 0.0989 - auc_2: 0.9331\n",
      "Epoch 2/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.3134 - accuracy: 0.8626 - mae: 0.1973 - mse: 0.0981 - auc_2: 0.9343\n",
      "Epoch 3/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.3097 - accuracy: 0.8647 - mae: 0.1941 - mse: 0.0968 - auc_2: 0.9359\n",
      "Epoch 4/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.3093 - accuracy: 0.8656 - mae: 0.1939 - mse: 0.0965 - auc_2: 0.9360\n",
      "Epoch 5/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.3065 - accuracy: 0.8668 - mae: 0.1921 - mse: 0.0958 - auc_2: 0.9371\n",
      "Epoch 6/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.3041 - accuracy: 0.8688 - mae: 0.1904 - mse: 0.0948 - auc_2: 0.9382\n",
      "Epoch 7/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.3054 - accuracy: 0.8674 - mae: 0.1910 - mse: 0.0952 - auc_2: 0.9377\n",
      "Epoch 8/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.3023 - accuracy: 0.8687 - mae: 0.1889 - mse: 0.0942 - auc_2: 0.9389\n",
      "Epoch 9/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.3011 - accuracy: 0.8691 - mae: 0.1884 - mse: 0.0940 - auc_2: 0.9394\n",
      "Epoch 10/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.2986 - accuracy: 0.8708 - mae: 0.1870 - mse: 0.0931 - auc_2: 0.9403\n",
      "86\n",
      "86\n",
      "86\n",
      "86\n",
      "86\n",
      "86\n",
      "86\n",
      "86\n",
      "86\n",
      "Epoch 1/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.2974 - accuracy: 0.8710 - mae: 0.1858 - mse: 0.0928 - auc_2: 0.9408\n",
      "Epoch 2/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.2957 - accuracy: 0.8722 - mae: 0.1843 - mse: 0.0922 - auc_2: 0.9414\n",
      "Epoch 3/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.2940 - accuracy: 0.8732 - mae: 0.1839 - mse: 0.0915 - auc_2: 0.9421\n",
      "Epoch 4/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.2957 - accuracy: 0.8727 - mae: 0.1846 - mse: 0.0919 - auc_2: 0.9415\n",
      "Epoch 5/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.2929 - accuracy: 0.8727 - mae: 0.1833 - mse: 0.0915 - auc_2: 0.9425\n",
      "Epoch 6/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.2918 - accuracy: 0.8726 - mae: 0.1824 - mse: 0.0908 - auc_2: 0.9432\n",
      "Epoch 7/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.2886 - accuracy: 0.8755 - mae: 0.1800 - mse: 0.0898 - auc_2: 0.9442\n",
      "Epoch 8/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.2935 - accuracy: 0.8735 - mae: 0.1836 - mse: 0.0912 - auc_2: 0.9424\n",
      "Epoch 9/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.2907 - accuracy: 0.8750 - mae: 0.1816 - mse: 0.0904 - auc_2: 0.9435\n",
      "Epoch 10/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.2841 - accuracy: 0.8785 - mae: 0.1767 - mse: 0.0881 - auc_2: 0.9459\n",
      "87\n",
      "87\n",
      "87\n",
      "87\n",
      "87\n",
      "87\n",
      "87\n",
      "87\n",
      "87\n",
      "5\n",
      "244/244 [==============================] - 0s 730us/step - loss: 0.2533 - accuracy: 0.8899 - mae: 0.1563 - mse: 0.0771 - auc_2: 0.9007\n",
      "Save_model\n",
      "train positive label: 5302 - train negative label: 64849\n",
      "up and down sampling => train positive label: 37114 - train negative label: 64849\n",
      "Test positive label: 560 - Test negative label: 7235\n",
      "Epoch 1/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.5251 - accuracy: 0.7492 - mae: 0.3468 - mse: 0.1725 - auc_3: 0.8001\n",
      "Epoch 2/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.4739 - accuracy: 0.7859 - mae: 0.3078 - mse: 0.1521 - auc_3: 0.8419\n",
      "Epoch 3/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.4600 - accuracy: 0.7900 - mae: 0.2977 - mse: 0.1476 - auc_3: 0.8505\n",
      "Epoch 4/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.4485 - accuracy: 0.7976 - mae: 0.2886 - mse: 0.1433 - auc_3: 0.8588\n",
      "Epoch 5/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.4398 - accuracy: 0.8023 - mae: 0.2827 - mse: 0.1404 - auc_3: 0.8645\n",
      "Epoch 6/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.4305 - accuracy: 0.8076 - mae: 0.2757 - mse: 0.1367 - auc_3: 0.8712\n",
      "Epoch 7/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.4236 - accuracy: 0.8099 - mae: 0.2714 - mse: 0.1345 - auc_3: 0.8757\n",
      "Epoch 8/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.4183 - accuracy: 0.8123 - mae: 0.2675 - mse: 0.1329 - auc_3: 0.8789\n",
      "Epoch 9/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.4104 - accuracy: 0.8159 - mae: 0.2622 - mse: 0.1301 - auc_3: 0.8842\n",
      "Epoch 10/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.4048 - accuracy: 0.8194 - mae: 0.2581 - mse: 0.1282 - auc_3: 0.8874\n",
      "81\n",
      "81\n",
      "80\n",
      "80\n",
      "80\n",
      "79\n",
      "78\n",
      "78\n",
      "74\n",
      "Epoch 1/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.3992 - accuracy: 0.8224 - mae: 0.2545 - mse: 0.1263 - auc_3: 0.8907\n",
      "Epoch 2/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.3910 - accuracy: 0.8258 - mae: 0.2489 - mse: 0.1238 - auc_3: 0.8952\n",
      "Epoch 3/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.3868 - accuracy: 0.8283 - mae: 0.2461 - mse: 0.1221 - auc_3: 0.8980\n",
      "Epoch 4/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.3784 - accuracy: 0.8316 - mae: 0.2405 - mse: 0.1194 - auc_3: 0.9024\n",
      "Epoch 5/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.3735 - accuracy: 0.8353 - mae: 0.2371 - mse: 0.1177 - auc_3: 0.9055\n",
      "Epoch 6/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.3692 - accuracy: 0.8368 - mae: 0.2343 - mse: 0.1164 - auc_3: 0.9074\n",
      "Epoch 7/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.3640 - accuracy: 0.8384 - mae: 0.2313 - mse: 0.1148 - auc_3: 0.9103\n",
      "Epoch 8/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.3588 - accuracy: 0.8421 - mae: 0.2268 - mse: 0.1128 - auc_3: 0.9132\n",
      "Epoch 9/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.3552 - accuracy: 0.8423 - mae: 0.2256 - mse: 0.1119 - auc_3: 0.9151\n",
      "Epoch 10/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.3514 - accuracy: 0.8442 - mae: 0.2223 - mse: 0.1105 - auc_3: 0.9170\n",
      "84\n",
      "84\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "82\n",
      "82\n",
      "82\n",
      "Epoch 1/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.3446 - accuracy: 0.8477 - mae: 0.2175 - mse: 0.1085 - auc_3: 0.9202\n",
      "Epoch 2/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.3409 - accuracy: 0.8499 - mae: 0.2153 - mse: 0.1070 - auc_3: 0.9221\n",
      "Epoch 3/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.3357 - accuracy: 0.8528 - mae: 0.2110 - mse: 0.1051 - auc_3: 0.9246\n",
      "Epoch 4/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.3308 - accuracy: 0.8546 - mae: 0.2086 - mse: 0.1037 - auc_3: 0.9268\n",
      "Epoch 5/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.3282 - accuracy: 0.8554 - mae: 0.2065 - mse: 0.1028 - auc_3: 0.9280\n",
      "Epoch 6/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.3245 - accuracy: 0.8567 - mae: 0.2040 - mse: 0.1015 - auc_3: 0.9298\n",
      "Epoch 7/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.3212 - accuracy: 0.8590 - mae: 0.2016 - mse: 0.1001 - auc_3: 0.9314\n",
      "Epoch 8/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.3176 - accuracy: 0.8603 - mae: 0.1991 - mse: 0.0992 - auc_3: 0.9327\n",
      "Epoch 9/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.3119 - accuracy: 0.8642 - mae: 0.1954 - mse: 0.0971 - auc_3: 0.9353\n",
      "Epoch 10/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.3092 - accuracy: 0.8657 - mae: 0.1934 - mse: 0.0963 - auc_3: 0.9363\n",
      "86\n",
      "86\n",
      "85\n",
      "85\n",
      "85\n",
      "85\n",
      "85\n",
      "84\n",
      "84\n",
      "Epoch 1/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.3065 - accuracy: 0.8679 - mae: 0.1917 - mse: 0.0954 - auc_3: 0.9375\n",
      "Epoch 2/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.3036 - accuracy: 0.8686 - mae: 0.1897 - mse: 0.0945 - auc_3: 0.9387\n",
      "Epoch 3/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.3012 - accuracy: 0.8699 - mae: 0.1880 - mse: 0.0935 - auc_3: 0.9398\n",
      "Epoch 4/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.2985 - accuracy: 0.8711 - mae: 0.1860 - mse: 0.0926 - auc_3: 0.9408\n",
      "Epoch 5/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.2943 - accuracy: 0.8731 - mae: 0.1830 - mse: 0.0911 - auc_3: 0.9426\n",
      "Epoch 6/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.2912 - accuracy: 0.8746 - mae: 0.1813 - mse: 0.0903 - auc_3: 0.9437\n",
      "Epoch 7/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.2891 - accuracy: 0.8751 - mae: 0.1799 - mse: 0.0896 - auc_3: 0.9445\n",
      "Epoch 8/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.2891 - accuracy: 0.8759 - mae: 0.1796 - mse: 0.0895 - auc_3: 0.9445\n",
      "Epoch 9/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.2854 - accuracy: 0.8769 - mae: 0.1776 - mse: 0.0884 - auc_3: 0.9459\n",
      "Epoch 10/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.2833 - accuracy: 0.8796 - mae: 0.1758 - mse: 0.0874 - auc_3: 0.9468\n",
      "87\n",
      "87\n",
      "87\n",
      "87\n",
      "87\n",
      "87\n",
      "86\n",
      "86\n",
      "86\n",
      "Epoch 1/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.2793 - accuracy: 0.8811 - mae: 0.1730 - mse: 0.0861 - auc_3: 0.9483\n",
      "Epoch 2/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.2784 - accuracy: 0.8810 - mae: 0.1726 - mse: 0.0858 - auc_3: 0.9486\n",
      "Epoch 3/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.2753 - accuracy: 0.8822 - mae: 0.1703 - mse: 0.0850 - auc_3: 0.9497\n",
      "Epoch 4/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.2703 - accuracy: 0.8839 - mae: 0.1675 - mse: 0.0834 - auc_3: 0.9516\n",
      "Epoch 5/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.2701 - accuracy: 0.8849 - mae: 0.1672 - mse: 0.0832 - auc_3: 0.9516\n",
      "Epoch 6/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.2701 - accuracy: 0.8857 - mae: 0.1664 - mse: 0.0829 - auc_3: 0.9515\n",
      "Epoch 7/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.2682 - accuracy: 0.8855 - mae: 0.1658 - mse: 0.0826 - auc_3: 0.9521\n",
      "Epoch 8/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.2668 - accuracy: 0.8870 - mae: 0.1643 - mse: 0.0818 - auc_3: 0.9527\n",
      "Epoch 9/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.2657 - accuracy: 0.8876 - mae: 0.1636 - mse: 0.0816 - auc_3: 0.9530\n",
      "Epoch 10/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.2643 - accuracy: 0.8882 - mae: 0.1631 - mse: 0.0812 - auc_3: 0.9535\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "5\n",
      "244/244 [==============================] - 0s 681us/step - loss: 0.2716 - accuracy: 0.8881 - mae: 0.1749 - mse: 0.0820 - auc_3: 0.8800\n",
      "train positive label: 5233 - train negative label: 64918\n",
      "up and down sampling => train positive label: 36631 - train negative label: 64918\n",
      "Test positive label: 629 - Test negative label: 7166\n",
      "Epoch 1/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.5337 - accuracy: 0.7406 - mae: 0.3523 - mse: 0.1759 - auc_4: 0.7914\n",
      "Epoch 2/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.4776 - accuracy: 0.7830 - mae: 0.3105 - mse: 0.1537 - auc_4: 0.8378\n",
      "Epoch 3/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.4606 - accuracy: 0.7928 - mae: 0.2972 - mse: 0.1476 - auc_4: 0.8493\n",
      "Epoch 4/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.4496 - accuracy: 0.7982 - mae: 0.2893 - mse: 0.1436 - auc_4: 0.8574\n",
      "Epoch 5/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.4381 - accuracy: 0.8041 - mae: 0.2815 - mse: 0.1397 - auc_4: 0.8650\n",
      "Epoch 6/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.4300 - accuracy: 0.8081 - mae: 0.2760 - mse: 0.1368 - auc_4: 0.8703\n",
      "Epoch 7/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.4243 - accuracy: 0.8115 - mae: 0.2715 - mse: 0.1349 - auc_4: 0.8741\n",
      "Epoch 8/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.4171 - accuracy: 0.8136 - mae: 0.2666 - mse: 0.1326 - auc_4: 0.8786\n",
      "Epoch 9/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.4121 - accuracy: 0.8173 - mae: 0.2633 - mse: 0.1306 - auc_4: 0.8824\n",
      "Epoch 10/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.4055 - accuracy: 0.8200 - mae: 0.2590 - mse: 0.1285 - auc_4: 0.8864\n",
      "81\n",
      "81\n",
      "81\n",
      "80\n",
      "80\n",
      "79\n",
      "79\n",
      "78\n",
      "74\n",
      "Epoch 1/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.3986 - accuracy: 0.8232 - mae: 0.2538 - mse: 0.1259 - auc_4: 0.8905\n",
      "Epoch 2/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.3956 - accuracy: 0.8251 - mae: 0.2519 - mse: 0.1250 - auc_4: 0.8924\n",
      "Epoch 3/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.3886 - accuracy: 0.8271 - mae: 0.2473 - mse: 0.1230 - auc_4: 0.8963\n",
      "Epoch 4/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.3843 - accuracy: 0.8295 - mae: 0.2443 - mse: 0.1215 - auc_4: 0.8989\n",
      "Epoch 5/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.3785 - accuracy: 0.8316 - mae: 0.2401 - mse: 0.1196 - auc_4: 0.9020\n",
      "Epoch 6/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.3730 - accuracy: 0.8344 - mae: 0.2369 - mse: 0.1176 - auc_4: 0.9053\n",
      "Epoch 7/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.3689 - accuracy: 0.8369 - mae: 0.2339 - mse: 0.1165 - auc_4: 0.9072\n",
      "Epoch 8/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.3631 - accuracy: 0.8398 - mae: 0.2301 - mse: 0.1144 - auc_4: 0.9103\n",
      "Epoch 9/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.3596 - accuracy: 0.8408 - mae: 0.2279 - mse: 0.1134 - auc_4: 0.9122\n",
      "Epoch 10/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.3544 - accuracy: 0.8431 - mae: 0.2239 - mse: 0.1116 - auc_4: 0.9149\n",
      "84\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "82\n",
      "82\n",
      "82\n",
      "82\n",
      "Epoch 1/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.3497 - accuracy: 0.8452 - mae: 0.2211 - mse: 0.1103 - auc_4: 0.9172\n",
      "Epoch 2/10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "794/794 [==============================] - 4s 5ms/step - loss: 0.3484 - accuracy: 0.8465 - mae: 0.2199 - mse: 0.1097 - auc_4: 0.9179\n",
      "Epoch 3/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.3444 - accuracy: 0.8485 - mae: 0.2179 - mse: 0.1082 - auc_4: 0.9201\n",
      "Epoch 4/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.3383 - accuracy: 0.8514 - mae: 0.2132 - mse: 0.1061 - auc_4: 0.9230\n",
      "Epoch 5/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.3347 - accuracy: 0.8523 - mae: 0.2109 - mse: 0.1053 - auc_4: 0.9245\n",
      "Epoch 6/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.3313 - accuracy: 0.8545 - mae: 0.2084 - mse: 0.1040 - auc_4: 0.9263\n",
      "Epoch 7/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.3297 - accuracy: 0.8556 - mae: 0.2077 - mse: 0.1033 - auc_4: 0.9270\n",
      "Epoch 8/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.3240 - accuracy: 0.8576 - mae: 0.2032 - mse: 0.1012 - auc_4: 0.9297\n",
      "Epoch 9/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.3221 - accuracy: 0.8592 - mae: 0.2021 - mse: 0.1007 - auc_4: 0.9304\n",
      "Epoch 10/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.3193 - accuracy: 0.8608 - mae: 0.1999 - mse: 0.0997 - auc_4: 0.9317\n",
      "85\n",
      "85\n",
      "85\n",
      "85\n",
      "85\n",
      "85\n",
      "84\n",
      "84\n",
      "84\n",
      "Epoch 1/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.3144 - accuracy: 0.8633 - mae: 0.1967 - mse: 0.0980 - auc_4: 0.9338\n",
      "Epoch 2/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.3159 - accuracy: 0.8625 - mae: 0.1981 - mse: 0.0987 - auc_4: 0.9331\n",
      "Epoch 3/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.3101 - accuracy: 0.8653 - mae: 0.1937 - mse: 0.0965 - auc_4: 0.9357\n",
      "Epoch 4/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.3076 - accuracy: 0.8652 - mae: 0.1922 - mse: 0.0959 - auc_4: 0.9367\n",
      "Epoch 5/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.3034 - accuracy: 0.8674 - mae: 0.1897 - mse: 0.0947 - auc_4: 0.9384\n",
      "Epoch 6/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.3010 - accuracy: 0.8693 - mae: 0.1878 - mse: 0.0935 - auc_4: 0.9395\n",
      "Epoch 7/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2985 - accuracy: 0.8701 - mae: 0.1863 - mse: 0.0930 - auc_4: 0.9404\n",
      "Epoch 8/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2971 - accuracy: 0.8710 - mae: 0.1855 - mse: 0.0925 - auc_4: 0.9410\n",
      "Epoch 9/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2961 - accuracy: 0.8730 - mae: 0.1838 - mse: 0.0918 - auc_4: 0.9415\n",
      "Epoch 10/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2918 - accuracy: 0.8730 - mae: 0.1821 - mse: 0.0908 - auc_4: 0.9431\n",
      "87\n",
      "87\n",
      "87\n",
      "86\n",
      "86\n",
      "86\n",
      "86\n",
      "86\n",
      "86\n",
      "Epoch 1/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2940 - accuracy: 0.8736 - mae: 0.1828 - mse: 0.0911 - auc_4: 0.9422\n",
      "Epoch 2/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2915 - accuracy: 0.8752 - mae: 0.1810 - mse: 0.0902 - auc_4: 0.9433\n",
      "Epoch 3/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2896 - accuracy: 0.8744 - mae: 0.1804 - mse: 0.0898 - auc_4: 0.9440\n",
      "Epoch 4/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2888 - accuracy: 0.8756 - mae: 0.1794 - mse: 0.0895 - auc_4: 0.9443\n",
      "Epoch 5/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2859 - accuracy: 0.8767 - mae: 0.1775 - mse: 0.0886 - auc_4: 0.9455\n",
      "Epoch 6/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2844 - accuracy: 0.8780 - mae: 0.1767 - mse: 0.0879 - auc_4: 0.9461\n",
      "Epoch 7/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2836 - accuracy: 0.8794 - mae: 0.1753 - mse: 0.0875 - auc_4: 0.9463\n",
      "Epoch 8/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2832 - accuracy: 0.8787 - mae: 0.1757 - mse: 0.0876 - auc_4: 0.9464\n",
      "Epoch 9/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2806 - accuracy: 0.8801 - mae: 0.1738 - mse: 0.0867 - auc_4: 0.9474\n",
      "Epoch 10/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2782 - accuracy: 0.8806 - mae: 0.1724 - mse: 0.0860 - auc_4: 0.9483\n",
      "88\n",
      "87\n",
      "87\n",
      "87\n",
      "87\n",
      "87\n",
      "87\n",
      "87\n",
      "87\n",
      "Epoch 1/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2777 - accuracy: 0.8808 - mae: 0.1723 - mse: 0.0858 - auc_4: 0.9485\n",
      "Epoch 2/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2785 - accuracy: 0.8812 - mae: 0.1723 - mse: 0.0859 - auc_4: 0.9482\n",
      "Epoch 3/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2756 - accuracy: 0.8814 - mae: 0.1710 - mse: 0.0853 - auc_4: 0.9492\n",
      "Epoch 4/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2797 - accuracy: 0.8807 - mae: 0.1729 - mse: 0.0863 - auc_4: 0.9478\n",
      "Epoch 5/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2773 - accuracy: 0.8827 - mae: 0.1715 - mse: 0.0855 - auc_4: 0.9485\n",
      "Epoch 6/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2784 - accuracy: 0.8804 - mae: 0.1729 - mse: 0.0860 - auc_4: 0.9483\n",
      "Epoch 7/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2714 - accuracy: 0.8842 - mae: 0.1678 - mse: 0.0837 - auc_4: 0.9508\n",
      "Epoch 8/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2723 - accuracy: 0.8842 - mae: 0.1682 - mse: 0.0840 - auc_4: 0.9503\n",
      "Epoch 9/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2719 - accuracy: 0.8836 - mae: 0.1680 - mse: 0.0837 - auc_4: 0.9506\n",
      "Epoch 10/10\n",
      "794/794 [==============================] - 4s 5ms/step - loss: 0.2728 - accuracy: 0.8836 - mae: 0.1688 - mse: 0.0842 - auc_4: 0.9502\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "6\n",
      "244/244 [==============================] - 0s 718us/step - loss: 0.2562 - accuracy: 0.8899 - mae: 0.1479 - mse: 0.0758 - auc_4: 0.8951\n",
      "train positive label: 5279 - train negative label: 64872\n",
      "up and down sampling => train positive label: 36953 - train negative label: 64872\n",
      "Test positive label: 583 - Test negative label: 7212\n",
      "Epoch 1/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.5149 - accuracy: 0.7587 - mae: 0.3390 - mse: 0.1680 - auc_5: 0.8099\n",
      "Epoch 2/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.4682 - accuracy: 0.7860 - mae: 0.3041 - mse: 0.1502 - auc_5: 0.8461\n",
      "Epoch 3/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.4523 - accuracy: 0.7970 - mae: 0.2919 - mse: 0.1443 - auc_5: 0.8572\n",
      "Epoch 4/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.4380 - accuracy: 0.8033 - mae: 0.2820 - mse: 0.1395 - auc_5: 0.8661\n",
      "Epoch 5/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.4281 - accuracy: 0.8093 - mae: 0.2750 - mse: 0.1360 - auc_5: 0.8732\n",
      "Epoch 6/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.4202 - accuracy: 0.8125 - mae: 0.2695 - mse: 0.1333 - auc_5: 0.8783\n",
      "Epoch 7/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.4135 - accuracy: 0.8152 - mae: 0.2647 - mse: 0.1310 - auc_5: 0.8826\n",
      "Epoch 8/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.4049 - accuracy: 0.8208 - mae: 0.2590 - mse: 0.1282 - auc_5: 0.8872\n",
      "Epoch 9/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.3967 - accuracy: 0.8234 - mae: 0.2534 - mse: 0.1255 - auc_5: 0.8925\n",
      "Epoch 10/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.3916 - accuracy: 0.8260 - mae: 0.2500 - mse: 0.1238 - auc_5: 0.8955\n",
      "82\n",
      "82\n",
      "81\n",
      "81\n",
      "80\n",
      "80\n",
      "79\n",
      "78\n",
      "75\n",
      "Epoch 1/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.3845 - accuracy: 0.8281 - mae: 0.2456 - mse: 0.1217 - auc_5: 0.8991\n",
      "Epoch 2/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.3800 - accuracy: 0.8313 - mae: 0.2420 - mse: 0.1200 - auc_5: 0.9019\n",
      "Epoch 3/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.3716 - accuracy: 0.8356 - mae: 0.2367 - mse: 0.1173 - auc_5: 0.9064\n",
      "Epoch 4/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.3671 - accuracy: 0.8378 - mae: 0.2337 - mse: 0.1158 - auc_5: 0.9089\n",
      "Epoch 5/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.3633 - accuracy: 0.8396 - mae: 0.2308 - mse: 0.1145 - auc_5: 0.9110\n",
      "Epoch 6/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.3556 - accuracy: 0.8437 - mae: 0.2258 - mse: 0.1119 - auc_5: 0.9148\n",
      "Epoch 7/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.3498 - accuracy: 0.8453 - mae: 0.2217 - mse: 0.1101 - auc_5: 0.9177\n",
      "Epoch 8/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.3454 - accuracy: 0.8471 - mae: 0.2190 - mse: 0.1086 - auc_5: 0.9201\n",
      "Epoch 9/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.3385 - accuracy: 0.8515 - mae: 0.2143 - mse: 0.1062 - auc_5: 0.9232\n",
      "Epoch 10/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.3366 - accuracy: 0.8514 - mae: 0.2126 - mse: 0.1057 - auc_5: 0.9241\n",
      "85\n",
      "84\n",
      "84\n",
      "84\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "82\n",
      "Epoch 1/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.3321 - accuracy: 0.8540 - mae: 0.2097 - mse: 0.1040 - auc_5: 0.9264\n",
      "Epoch 2/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.3262 - accuracy: 0.8558 - mae: 0.2055 - mse: 0.1023 - auc_5: 0.9289\n",
      "Epoch 3/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.3213 - accuracy: 0.8595 - mae: 0.2026 - mse: 0.1006 - auc_5: 0.9311\n",
      "Epoch 4/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.3178 - accuracy: 0.8617 - mae: 0.1996 - mse: 0.0990 - auc_5: 0.9328\n",
      "Epoch 5/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.3130 - accuracy: 0.8631 - mae: 0.1961 - mse: 0.0978 - auc_5: 0.9346\n",
      "Epoch 6/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.3092 - accuracy: 0.8665 - mae: 0.1937 - mse: 0.0961 - auc_5: 0.9363\n",
      "Epoch 7/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.3076 - accuracy: 0.8663 - mae: 0.1926 - mse: 0.0959 - auc_5: 0.9370\n",
      "Epoch 8/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.3041 - accuracy: 0.8670 - mae: 0.1906 - mse: 0.0949 - auc_5: 0.9385\n",
      "Epoch 9/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2996 - accuracy: 0.8699 - mae: 0.1877 - mse: 0.0933 - auc_5: 0.9402\n",
      "Epoch 10/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2980 - accuracy: 0.8707 - mae: 0.1858 - mse: 0.0926 - auc_5: 0.9409\n",
      "86\n",
      "86\n",
      "86\n",
      "86\n",
      "86\n",
      "86\n",
      "85\n",
      "85\n",
      "85\n",
      "Epoch 1/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2957 - accuracy: 0.8711 - mae: 0.1848 - mse: 0.0921 - auc_5: 0.9417\n",
      "Epoch 2/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2918 - accuracy: 0.8744 - mae: 0.1819 - mse: 0.0905 - auc_5: 0.9434\n",
      "Epoch 3/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2916 - accuracy: 0.8749 - mae: 0.1815 - mse: 0.0904 - auc_5: 0.9434\n",
      "Epoch 4/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2906 - accuracy: 0.8744 - mae: 0.1813 - mse: 0.0901 - auc_5: 0.9438\n",
      "Epoch 5/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2887 - accuracy: 0.8766 - mae: 0.1798 - mse: 0.0894 - auc_5: 0.9446\n",
      "Epoch 6/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2858 - accuracy: 0.8761 - mae: 0.1779 - mse: 0.0886 - auc_5: 0.9457\n",
      "Epoch 7/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2828 - accuracy: 0.8786 - mae: 0.1756 - mse: 0.0873 - auc_5: 0.9469\n",
      "Epoch 8/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2815 - accuracy: 0.8789 - mae: 0.1752 - mse: 0.0872 - auc_5: 0.9472\n",
      "Epoch 9/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2799 - accuracy: 0.8801 - mae: 0.1737 - mse: 0.0866 - auc_5: 0.9479\n",
      "Epoch 10/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2798 - accuracy: 0.8797 - mae: 0.1740 - mse: 0.0867 - auc_5: 0.9480\n",
      "88\n",
      "87\n",
      "87\n",
      "87\n",
      "87\n",
      "87\n",
      "87\n",
      "87\n",
      "87\n",
      "Epoch 1/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2786 - accuracy: 0.8809 - mae: 0.1728 - mse: 0.0861 - auc_5: 0.9483\n",
      "Epoch 2/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2763 - accuracy: 0.8817 - mae: 0.1712 - mse: 0.0852 - auc_5: 0.9493\n",
      "Epoch 3/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2748 - accuracy: 0.8824 - mae: 0.1705 - mse: 0.0850 - auc_5: 0.9496\n",
      "Epoch 4/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2729 - accuracy: 0.8834 - mae: 0.1690 - mse: 0.0842 - auc_5: 0.9504\n",
      "Epoch 5/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2733 - accuracy: 0.8819 - mae: 0.1698 - mse: 0.0845 - auc_5: 0.9503\n",
      "Epoch 6/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2720 - accuracy: 0.8840 - mae: 0.1685 - mse: 0.0838 - auc_5: 0.9507\n",
      "Epoch 7/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2745 - accuracy: 0.8819 - mae: 0.1700 - mse: 0.0847 - auc_5: 0.9500\n",
      "Epoch 8/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2726 - accuracy: 0.8834 - mae: 0.1690 - mse: 0.0841 - auc_5: 0.9505\n",
      "Epoch 9/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2672 - accuracy: 0.8857 - mae: 0.1650 - mse: 0.0824 - auc_5: 0.9525\n",
      "Epoch 10/10\n",
      "796/796 [==============================] - 4s 5ms/step - loss: 0.2701 - accuracy: 0.8836 - mae: 0.1672 - mse: 0.0835 - auc_5: 0.9514\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "5\n",
      "244/244 [==============================] - 0s 726us/step - loss: 0.2644 - accuracy: 0.8945 - mae: 0.1727 - mse: 0.0776 - auc_5: 0.8922\n",
      "train positive label: 5266 - train negative label: 64886\n",
      "up and down sampling => train positive label: 36862 - train negative label: 64886\n",
      "Test positive label: 596 - Test negative label: 7198\n",
      "Epoch 1/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.5231 - accuracy: 0.7481 - mae: 0.3454 - mse: 0.1717 - auc_6: 0.8026\n",
      "Epoch 2/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.4730 - accuracy: 0.7857 - mae: 0.3069 - mse: 0.1518 - auc_6: 0.8426\n",
      "Epoch 3/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.4568 - accuracy: 0.7939 - mae: 0.2954 - mse: 0.1461 - auc_6: 0.8532\n",
      "Epoch 4/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.4475 - accuracy: 0.8010 - mae: 0.2882 - mse: 0.1426 - auc_6: 0.8589\n",
      "Epoch 5/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.4362 - accuracy: 0.8049 - mae: 0.2801 - mse: 0.1387 - auc_6: 0.8673\n",
      "Epoch 6/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.4276 - accuracy: 0.8086 - mae: 0.2744 - mse: 0.1359 - auc_6: 0.8727\n",
      "Epoch 7/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.4197 - accuracy: 0.8131 - mae: 0.2689 - mse: 0.1334 - auc_6: 0.8776\n",
      "Epoch 8/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.4136 - accuracy: 0.8156 - mae: 0.2643 - mse: 0.1312 - auc_6: 0.8817\n",
      "Epoch 9/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.4081 - accuracy: 0.8177 - mae: 0.2606 - mse: 0.1294 - auc_6: 0.8852\n",
      "Epoch 10/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.4000 - accuracy: 0.8221 - mae: 0.2550 - mse: 0.1266 - auc_6: 0.8901\n",
      "81\n",
      "81\n",
      "81\n",
      "80\n",
      "80\n",
      "80\n",
      "79\n",
      "78\n",
      "74\n",
      "Epoch 1/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.3950 - accuracy: 0.8245 - mae: 0.2523 - mse: 0.1251 - auc_6: 0.8929\n",
      "Epoch 2/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.3905 - accuracy: 0.8258 - mae: 0.2489 - mse: 0.1237 - auc_6: 0.8956\n",
      "Epoch 3/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.3821 - accuracy: 0.8307 - mae: 0.2427 - mse: 0.1206 - auc_6: 0.9004\n",
      "Epoch 4/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.3798 - accuracy: 0.8318 - mae: 0.2411 - mse: 0.1197 - auc_6: 0.9018\n",
      "Epoch 5/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.3732 - accuracy: 0.8339 - mae: 0.2369 - mse: 0.1177 - auc_6: 0.9053\n",
      "Epoch 6/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.3681 - accuracy: 0.8374 - mae: 0.2329 - mse: 0.1157 - auc_6: 0.9081\n",
      "Epoch 7/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.3649 - accuracy: 0.8397 - mae: 0.2305 - mse: 0.1146 - auc_6: 0.9097\n",
      "Epoch 8/10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "795/795 [==============================] - 4s 5ms/step - loss: 0.3573 - accuracy: 0.8427 - mae: 0.2249 - mse: 0.1120 - auc_6: 0.9136\n",
      "Epoch 9/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.3564 - accuracy: 0.8430 - mae: 0.2251 - mse: 0.1120 - auc_6: 0.9142\n",
      "Epoch 10/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.3499 - accuracy: 0.8469 - mae: 0.2204 - mse: 0.1096 - auc_6: 0.9174\n",
      "84\n",
      "84\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "82\n",
      "82\n",
      "Epoch 1/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.3461 - accuracy: 0.8479 - mae: 0.2179 - mse: 0.1084 - auc_6: 0.9194\n",
      "Epoch 2/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.3428 - accuracy: 0.8505 - mae: 0.2152 - mse: 0.1071 - auc_6: 0.9211\n",
      "Epoch 3/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.3392 - accuracy: 0.8518 - mae: 0.2130 - mse: 0.1061 - auc_6: 0.9227\n",
      "Epoch 4/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.3338 - accuracy: 0.8541 - mae: 0.2092 - mse: 0.1044 - auc_6: 0.9252\n",
      "Epoch 5/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.3317 - accuracy: 0.8553 - mae: 0.2082 - mse: 0.1034 - auc_6: 0.9265\n",
      "Epoch 6/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.3281 - accuracy: 0.8576 - mae: 0.2055 - mse: 0.1022 - auc_6: 0.9280\n",
      "Epoch 7/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.3233 - accuracy: 0.8606 - mae: 0.2019 - mse: 0.1005 - auc_6: 0.9302\n",
      "Epoch 8/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.3219 - accuracy: 0.8596 - mae: 0.2021 - mse: 0.1006 - auc_6: 0.9306\n",
      "Epoch 9/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.3172 - accuracy: 0.8626 - mae: 0.1981 - mse: 0.0987 - auc_6: 0.9329\n",
      "Epoch 10/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.3161 - accuracy: 0.8629 - mae: 0.1971 - mse: 0.0982 - auc_6: 0.9334\n",
      "86\n",
      "85\n",
      "86\n",
      "85\n",
      "85\n",
      "85\n",
      "85\n",
      "85\n",
      "84\n",
      "Epoch 1/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.3107 - accuracy: 0.8662 - mae: 0.1935 - mse: 0.0963 - auc_6: 0.9356\n",
      "Epoch 2/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.3078 - accuracy: 0.8669 - mae: 0.1918 - mse: 0.0956 - auc_6: 0.9369\n",
      "Epoch 3/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.3054 - accuracy: 0.8676 - mae: 0.1901 - mse: 0.0947 - auc_6: 0.9379\n",
      "Epoch 4/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.3025 - accuracy: 0.8698 - mae: 0.1887 - mse: 0.0940 - auc_6: 0.9390\n",
      "Epoch 5/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.2983 - accuracy: 0.8719 - mae: 0.1855 - mse: 0.0924 - auc_6: 0.9408\n",
      "Epoch 6/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.3001 - accuracy: 0.8705 - mae: 0.1867 - mse: 0.0929 - auc_6: 0.9401\n",
      "Epoch 7/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.2961 - accuracy: 0.8724 - mae: 0.1842 - mse: 0.0917 - auc_6: 0.9416\n",
      "Epoch 8/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.2927 - accuracy: 0.8739 - mae: 0.1820 - mse: 0.0908 - auc_6: 0.9429\n",
      "Epoch 9/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.2916 - accuracy: 0.8752 - mae: 0.1808 - mse: 0.0900 - auc_6: 0.9435\n",
      "Epoch 10/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.2880 - accuracy: 0.8763 - mae: 0.1787 - mse: 0.0890 - auc_6: 0.9448\n",
      "87\n",
      "87\n",
      "87\n",
      "87\n",
      "87\n",
      "86\n",
      "86\n",
      "86\n",
      "86\n",
      "Epoch 1/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.2863 - accuracy: 0.8770 - mae: 0.1772 - mse: 0.0885 - auc_6: 0.9454\n",
      "Epoch 2/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.2837 - accuracy: 0.8791 - mae: 0.1758 - mse: 0.0876 - auc_6: 0.9464\n",
      "Epoch 3/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.2840 - accuracy: 0.8784 - mae: 0.1760 - mse: 0.0875 - auc_6: 0.9464\n",
      "Epoch 4/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.2843 - accuracy: 0.8781 - mae: 0.1758 - mse: 0.0877 - auc_6: 0.9462\n",
      "Epoch 5/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.2795 - accuracy: 0.8804 - mae: 0.1726 - mse: 0.0860 - auc_6: 0.9481\n",
      "Epoch 6/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.2766 - accuracy: 0.8828 - mae: 0.1711 - mse: 0.0851 - auc_6: 0.9490\n",
      "Epoch 7/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.2792 - accuracy: 0.8807 - mae: 0.1730 - mse: 0.0862 - auc_6: 0.9480\n",
      "Epoch 8/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.2755 - accuracy: 0.8817 - mae: 0.1702 - mse: 0.0851 - auc_6: 0.9492\n",
      "Epoch 9/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.2766 - accuracy: 0.8819 - mae: 0.1714 - mse: 0.0854 - auc_6: 0.9490\n",
      "Epoch 10/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.2737 - accuracy: 0.8835 - mae: 0.1688 - mse: 0.0842 - auc_6: 0.9501\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "87\n",
      "87\n",
      "87\n",
      "87\n",
      "Epoch 1/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.2718 - accuracy: 0.8844 - mae: 0.1681 - mse: 0.0836 - auc_6: 0.9508\n",
      "Epoch 2/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.2697 - accuracy: 0.8865 - mae: 0.1659 - mse: 0.0828 - auc_6: 0.9515\n",
      "Epoch 3/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.2697 - accuracy: 0.8849 - mae: 0.1664 - mse: 0.0829 - auc_6: 0.9515\n",
      "Epoch 4/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.2663 - accuracy: 0.8870 - mae: 0.1640 - mse: 0.0817 - auc_6: 0.9527\n",
      "Epoch 5/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.2645 - accuracy: 0.8881 - mae: 0.1630 - mse: 0.0812 - auc_6: 0.9534\n",
      "Epoch 6/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.2632 - accuracy: 0.8895 - mae: 0.1616 - mse: 0.0805 - auc_6: 0.9538\n",
      "Epoch 7/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.2663 - accuracy: 0.8868 - mae: 0.1633 - mse: 0.0815 - auc_6: 0.9527\n",
      "Epoch 8/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.2592 - accuracy: 0.8905 - mae: 0.1595 - mse: 0.0796 - auc_6: 0.9550\n",
      "Epoch 9/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.2590 - accuracy: 0.8909 - mae: 0.1591 - mse: 0.0793 - auc_6: 0.9551\n",
      "Epoch 10/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.2605 - accuracy: 0.8906 - mae: 0.1600 - mse: 0.0799 - auc_6: 0.9545\n",
      "89\n",
      "89\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "Epoch 1/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.2608 - accuracy: 0.8900 - mae: 0.1603 - mse: 0.0799 - auc_6: 0.9545\n",
      "Epoch 2/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.2617 - accuracy: 0.8884 - mae: 0.1613 - mse: 0.0804 - auc_6: 0.9543\n",
      "Epoch 3/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.2590 - accuracy: 0.8898 - mae: 0.1595 - mse: 0.0795 - auc_6: 0.9551\n",
      "Epoch 4/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.2613 - accuracy: 0.8903 - mae: 0.1599 - mse: 0.0798 - auc_6: 0.9543\n",
      "Epoch 5/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.2600 - accuracy: 0.8908 - mae: 0.1597 - mse: 0.0797 - auc_6: 0.9547\n",
      "Epoch 6/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.2602 - accuracy: 0.8898 - mae: 0.1599 - mse: 0.0799 - auc_6: 0.9546\n",
      "Epoch 7/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.2577 - accuracy: 0.8919 - mae: 0.1581 - mse: 0.0787 - auc_6: 0.9557\n",
      "Epoch 8/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.2609 - accuracy: 0.8900 - mae: 0.1603 - mse: 0.0799 - auc_6: 0.9545\n",
      "Epoch 9/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.2559 - accuracy: 0.8927 - mae: 0.1569 - mse: 0.0784 - auc_6: 0.9561\n",
      "Epoch 10/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.2578 - accuracy: 0.8923 - mae: 0.1579 - mse: 0.0788 - auc_6: 0.9554\n",
      "89\n",
      "88\n",
      "89\n",
      "88\n",
      "89\n",
      "89\n",
      "88\n",
      "88\n",
      "89\n",
      "Epoch 1/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.2524 - accuracy: 0.8945 - mae: 0.1547 - mse: 0.0772 - auc_6: 0.9571\n",
      "Epoch 2/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.2518 - accuracy: 0.8944 - mae: 0.1546 - mse: 0.0769 - auc_6: 0.9576\n",
      "Epoch 3/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.2532 - accuracy: 0.8937 - mae: 0.1552 - mse: 0.0775 - auc_6: 0.9570\n",
      "Epoch 4/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.2511 - accuracy: 0.8944 - mae: 0.1541 - mse: 0.0770 - auc_6: 0.9575\n",
      "Epoch 5/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.2484 - accuracy: 0.8955 - mae: 0.1522 - mse: 0.0759 - auc_6: 0.9587\n",
      "Epoch 6/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.2489 - accuracy: 0.8958 - mae: 0.1528 - mse: 0.0760 - auc_6: 0.9584\n",
      "Epoch 7/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.2504 - accuracy: 0.8951 - mae: 0.1537 - mse: 0.0765 - auc_6: 0.9580\n",
      "Epoch 8/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.2506 - accuracy: 0.8951 - mae: 0.1539 - mse: 0.0767 - auc_6: 0.9578\n",
      "Epoch 9/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.2490 - accuracy: 0.8948 - mae: 0.1528 - mse: 0.0761 - auc_6: 0.9585\n",
      "Epoch 10/10\n",
      "795/795 [==============================] - 4s 5ms/step - loss: 0.2440 - accuracy: 0.8977 - mae: 0.1495 - mse: 0.0746 - auc_6: 0.9598\n",
      "89\n",
      "89\n",
      "89\n",
      "89\n",
      "89\n",
      "89\n",
      "89\n",
      "89\n",
      "89\n",
      "8\n",
      "244/244 [==============================] - 0s 685us/step - loss: 0.2348 - accuracy: 0.9070 - mae: 0.1409 - mse: 0.0708 - auc_6: 0.9094\n",
      "Save_model\n",
      "train positive label: 5306 - train negative label: 64846\n",
      "up and down sampling => train positive label: 37142 - train negative label: 64846\n",
      "Test positive label: 556 - Test negative label: 7238\n",
      "Epoch 1/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.5434 - accuracy: 0.7423 - mae: 0.3658 - mse: 0.1802 - auc_7: 0.7819\n",
      "Epoch 2/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.4728 - accuracy: 0.7894 - mae: 0.3080 - mse: 0.1515 - auc_7: 0.8410\n",
      "Epoch 3/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.4534 - accuracy: 0.7986 - mae: 0.2927 - mse: 0.1446 - auc_7: 0.8539\n",
      "Epoch 4/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.4458 - accuracy: 0.7999 - mae: 0.2874 - mse: 0.1422 - auc_7: 0.8601\n",
      "Epoch 5/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.4350 - accuracy: 0.8061 - mae: 0.2787 - mse: 0.1384 - auc_7: 0.8673\n",
      "Epoch 6/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.4281 - accuracy: 0.8105 - mae: 0.2737 - mse: 0.1357 - auc_7: 0.8725\n",
      "Epoch 7/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.4203 - accuracy: 0.8136 - mae: 0.2677 - mse: 0.1329 - auc_7: 0.8774\n",
      "Epoch 8/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.4118 - accuracy: 0.8179 - mae: 0.2628 - mse: 0.1303 - auc_7: 0.8825\n",
      "Epoch 9/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.4067 - accuracy: 0.8195 - mae: 0.2586 - mse: 0.1286 - auc_7: 0.8859\n",
      "Epoch 10/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.3999 - accuracy: 0.8226 - mae: 0.2537 - mse: 0.1260 - auc_7: 0.8901\n",
      "81\n",
      "81\n",
      "81\n",
      "81\n",
      "80\n",
      "79\n",
      "79\n",
      "78\n",
      "74\n",
      "Epoch 1/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.3946 - accuracy: 0.8249 - mae: 0.2501 - mse: 0.1245 - auc_7: 0.8933\n",
      "Epoch 2/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.3908 - accuracy: 0.8278 - mae: 0.2474 - mse: 0.1231 - auc_7: 0.8956\n",
      "Epoch 3/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.3838 - accuracy: 0.8316 - mae: 0.2423 - mse: 0.1206 - auc_7: 0.8997\n",
      "Epoch 4/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.3790 - accuracy: 0.8331 - mae: 0.2394 - mse: 0.1190 - auc_7: 0.9023\n",
      "Epoch 5/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.3730 - accuracy: 0.8367 - mae: 0.2350 - mse: 0.1171 - auc_7: 0.9053\n",
      "Epoch 6/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.3691 - accuracy: 0.8388 - mae: 0.2329 - mse: 0.1155 - auc_7: 0.9077\n",
      "Epoch 7/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.3650 - accuracy: 0.8393 - mae: 0.2297 - mse: 0.1145 - auc_7: 0.9098\n",
      "Epoch 8/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.3617 - accuracy: 0.8415 - mae: 0.2278 - mse: 0.1133 - auc_7: 0.9119\n",
      "Epoch 9/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.3573 - accuracy: 0.8451 - mae: 0.2245 - mse: 0.1116 - auc_7: 0.9140\n",
      "Epoch 10/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.3543 - accuracy: 0.8455 - mae: 0.2233 - mse: 0.1108 - auc_7: 0.9156\n",
      "84\n",
      "84\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "82\n",
      "82\n",
      "Epoch 1/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.3473 - accuracy: 0.8489 - mae: 0.2183 - mse: 0.1085 - auc_7: 0.9191\n",
      "Epoch 2/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.3452 - accuracy: 0.8498 - mae: 0.2162 - mse: 0.1077 - auc_7: 0.9202\n",
      "Epoch 3/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.3432 - accuracy: 0.8501 - mae: 0.2156 - mse: 0.1073 - auc_7: 0.9212\n",
      "Epoch 4/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.3398 - accuracy: 0.8529 - mae: 0.2125 - mse: 0.1059 - auc_7: 0.9227\n",
      "Epoch 5/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.3362 - accuracy: 0.8553 - mae: 0.2102 - mse: 0.1046 - auc_7: 0.9244\n",
      "Epoch 6/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.3327 - accuracy: 0.8563 - mae: 0.2084 - mse: 0.1038 - auc_7: 0.9262\n",
      "Epoch 7/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.3313 - accuracy: 0.8566 - mae: 0.2072 - mse: 0.1030 - auc_7: 0.9268\n",
      "Epoch 8/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.3240 - accuracy: 0.8594 - mae: 0.2025 - mse: 0.1009 - auc_7: 0.9301\n",
      "Epoch 9/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.3214 - accuracy: 0.8610 - mae: 0.2007 - mse: 0.0999 - auc_7: 0.9313\n",
      "Epoch 10/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.3214 - accuracy: 0.8614 - mae: 0.2007 - mse: 0.0999 - auc_7: 0.9314\n",
      "86\n",
      "85\n",
      "85\n",
      "85\n",
      "85\n",
      "85\n",
      "85\n",
      "84\n",
      "84\n",
      "Epoch 1/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.3168 - accuracy: 0.8643 - mae: 0.1977 - mse: 0.0982 - auc_7: 0.9333\n",
      "Epoch 2/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.3164 - accuracy: 0.8640 - mae: 0.1968 - mse: 0.0981 - auc_7: 0.9334\n",
      "Epoch 3/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.3142 - accuracy: 0.8655 - mae: 0.1956 - mse: 0.0973 - auc_7: 0.9346\n",
      "Epoch 4/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.3094 - accuracy: 0.8678 - mae: 0.1923 - mse: 0.0958 - auc_7: 0.9366\n",
      "Epoch 5/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.3078 - accuracy: 0.8678 - mae: 0.1919 - mse: 0.0955 - auc_7: 0.9372\n",
      "Epoch 6/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.3053 - accuracy: 0.8683 - mae: 0.1901 - mse: 0.0945 - auc_7: 0.9383\n",
      "Epoch 7/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.3019 - accuracy: 0.8701 - mae: 0.1875 - mse: 0.0934 - auc_7: 0.9399\n",
      "Epoch 8/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.3029 - accuracy: 0.8704 - mae: 0.1881 - mse: 0.0936 - auc_7: 0.9393\n",
      "Epoch 9/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.2988 - accuracy: 0.8727 - mae: 0.1859 - mse: 0.0923 - auc_7: 0.9409\n",
      "Epoch 10/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.2991 - accuracy: 0.8719 - mae: 0.1853 - mse: 0.0924 - auc_7: 0.9407\n",
      "87\n",
      "87\n",
      "87\n",
      "86\n",
      "86\n",
      "86\n",
      "86\n",
      "86\n",
      "86\n",
      "Epoch 1/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.2955 - accuracy: 0.8736 - mae: 0.1837 - mse: 0.0914 - auc_7: 0.9424\n",
      "Epoch 2/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.2945 - accuracy: 0.8727 - mae: 0.1828 - mse: 0.0911 - auc_7: 0.9426\n",
      "Epoch 3/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.2962 - accuracy: 0.8732 - mae: 0.1838 - mse: 0.0914 - auc_7: 0.9420\n",
      "Epoch 4/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.2889 - accuracy: 0.8763 - mae: 0.1789 - mse: 0.0893 - auc_7: 0.9448\n",
      "Epoch 5/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.2863 - accuracy: 0.8777 - mae: 0.1771 - mse: 0.0883 - auc_7: 0.9459\n",
      "Epoch 6/10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "797/797 [==============================] - 5s 6ms/step - loss: 0.2858 - accuracy: 0.8783 - mae: 0.1765 - mse: 0.0879 - auc_7: 0.9461\n",
      "Epoch 7/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.2871 - accuracy: 0.8779 - mae: 0.1776 - mse: 0.0882 - auc_7: 0.9457\n",
      "Epoch 8/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.2849 - accuracy: 0.8784 - mae: 0.1763 - mse: 0.0877 - auc_7: 0.9464\n",
      "Epoch 9/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.2838 - accuracy: 0.8795 - mae: 0.1753 - mse: 0.0874 - auc_7: 0.9467\n",
      "Epoch 10/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.2789 - accuracy: 0.8819 - mae: 0.1727 - mse: 0.0856 - auc_7: 0.9488\n",
      "87\n",
      "87\n",
      "87\n",
      "87\n",
      "87\n",
      "87\n",
      "87\n",
      "87\n",
      "87\n",
      "Epoch 1/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.2815 - accuracy: 0.8808 - mae: 0.1735 - mse: 0.0865 - auc_7: 0.9477\n",
      "Epoch 2/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.2804 - accuracy: 0.8814 - mae: 0.1733 - mse: 0.0862 - auc_7: 0.9482\n",
      "Epoch 3/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.2813 - accuracy: 0.8796 - mae: 0.1735 - mse: 0.0865 - auc_7: 0.9478\n",
      "Epoch 4/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.2779 - accuracy: 0.8821 - mae: 0.1712 - mse: 0.0855 - auc_7: 0.9490\n",
      "Epoch 5/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.2805 - accuracy: 0.8805 - mae: 0.1735 - mse: 0.0863 - auc_7: 0.9479\n",
      "Epoch 6/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.2793 - accuracy: 0.8803 - mae: 0.1726 - mse: 0.0859 - auc_7: 0.9485\n",
      "Epoch 7/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.2829 - accuracy: 0.8783 - mae: 0.1753 - mse: 0.0873 - auc_7: 0.9471\n",
      "Epoch 8/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.2784 - accuracy: 0.8813 - mae: 0.1718 - mse: 0.0857 - auc_7: 0.9487\n",
      "Epoch 9/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.2782 - accuracy: 0.8817 - mae: 0.1722 - mse: 0.0856 - auc_7: 0.9489\n",
      "Epoch 10/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.2740 - accuracy: 0.8829 - mae: 0.1693 - mse: 0.0844 - auc_7: 0.9504\n",
      "88\n",
      "88\n",
      "87\n",
      "88\n",
      "88\n",
      "88\n",
      "87\n",
      "88\n",
      "88\n",
      "Epoch 1/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.2765 - accuracy: 0.8822 - mae: 0.1705 - mse: 0.0850 - auc_7: 0.9494\n",
      "Epoch 2/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.2730 - accuracy: 0.8831 - mae: 0.1689 - mse: 0.0841 - auc_7: 0.9507\n",
      "Epoch 3/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.2751 - accuracy: 0.8826 - mae: 0.1700 - mse: 0.0846 - auc_7: 0.9500\n",
      "Epoch 4/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.2703 - accuracy: 0.8846 - mae: 0.1671 - mse: 0.0832 - auc_7: 0.9517\n",
      "Epoch 5/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.2734 - accuracy: 0.8844 - mae: 0.1684 - mse: 0.0840 - auc_7: 0.9505\n",
      "Epoch 6/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.2693 - accuracy: 0.8845 - mae: 0.1665 - mse: 0.0829 - auc_7: 0.9521\n",
      "Epoch 7/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.2652 - accuracy: 0.8876 - mae: 0.1635 - mse: 0.0813 - auc_7: 0.9535\n",
      "Epoch 8/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.2684 - accuracy: 0.8852 - mae: 0.1658 - mse: 0.0824 - auc_7: 0.9524\n",
      "Epoch 9/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.2653 - accuracy: 0.8866 - mae: 0.1636 - mse: 0.0816 - auc_7: 0.9533\n",
      "Epoch 10/10\n",
      "797/797 [==============================] - 5s 6ms/step - loss: 0.2666 - accuracy: 0.8863 - mae: 0.1641 - mse: 0.0819 - auc_7: 0.9528\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "7\n",
      "244/244 [==============================] - 0s 738us/step - loss: 0.2235 - accuracy: 0.9018 - mae: 0.1448 - mse: 0.0671 - auc_7: 0.8915\n",
      "train positive label: 5312 - train negative label: 64840\n",
      "up and down sampling => train positive label: 37184 - train negative label: 64840\n",
      "Test positive label: 550 - Test negative label: 7244\n",
      "Epoch 1/10\n",
      "798/798 [==============================] - 4s 6ms/step - loss: 0.5385 - accuracy: 0.7342 - mae: 0.3596 - mse: 0.1782 - auc_8: 0.7872\n",
      "Epoch 2/10\n",
      "798/798 [==============================] - 4s 5ms/step - loss: 0.4739 - accuracy: 0.7848 - mae: 0.3083 - mse: 0.1523 - auc_8: 0.8416\n",
      "Epoch 3/10\n",
      "798/798 [==============================] - 4s 6ms/step - loss: 0.4552 - accuracy: 0.7933 - mae: 0.2938 - mse: 0.1458 - auc_8: 0.8542\n",
      "Epoch 4/10\n",
      "798/798 [==============================] - 4s 5ms/step - loss: 0.4445 - accuracy: 0.8004 - mae: 0.2854 - mse: 0.1418 - auc_8: 0.8617\n",
      "Epoch 5/10\n",
      "798/798 [==============================] - 4s 5ms/step - loss: 0.4327 - accuracy: 0.8051 - mae: 0.2776 - mse: 0.1379 - auc_8: 0.8692\n",
      "Epoch 6/10\n",
      "798/798 [==============================] - 4s 6ms/step - loss: 0.4259 - accuracy: 0.8089 - mae: 0.2728 - mse: 0.1353 - auc_8: 0.8739\n",
      "Epoch 7/10\n",
      "798/798 [==============================] - 4s 6ms/step - loss: 0.4185 - accuracy: 0.8142 - mae: 0.2672 - mse: 0.1327 - auc_8: 0.8790\n",
      "Epoch 8/10\n",
      "798/798 [==============================] - 4s 5ms/step - loss: 0.4123 - accuracy: 0.8162 - mae: 0.2632 - mse: 0.1307 - auc_8: 0.8826\n",
      "Epoch 9/10\n",
      "798/798 [==============================] - 4s 5ms/step - loss: 0.4046 - accuracy: 0.8195 - mae: 0.2579 - mse: 0.1282 - auc_8: 0.8872\n",
      "Epoch 10/10\n",
      "798/798 [==============================] - 4s 6ms/step - loss: 0.3971 - accuracy: 0.8241 - mae: 0.2526 - mse: 0.1255 - auc_8: 0.8919\n",
      "81\n",
      "81\n",
      "81\n",
      "80\n",
      "80\n",
      "80\n",
      "79\n",
      "78\n",
      "73\n",
      "Epoch 1/10\n",
      "798/798 [==============================] - 4s 6ms/step - loss: 0.3931 - accuracy: 0.8258 - mae: 0.2494 - mse: 0.1240 - auc_8: 0.8945\n",
      "Epoch 2/10\n",
      "798/798 [==============================] - 4s 6ms/step - loss: 0.3853 - accuracy: 0.8284 - mae: 0.2445 - mse: 0.1216 - auc_8: 0.8987\n",
      "Epoch 3/10\n",
      "798/798 [==============================] - 4s 6ms/step - loss: 0.3832 - accuracy: 0.8314 - mae: 0.2427 - mse: 0.1207 - auc_8: 0.9002\n",
      "Epoch 4/10\n",
      "798/798 [==============================] - 4s 6ms/step - loss: 0.3763 - accuracy: 0.8331 - mae: 0.2385 - mse: 0.1186 - auc_8: 0.9038\n",
      "Epoch 5/10\n",
      "798/798 [==============================] - 4s 6ms/step - loss: 0.3715 - accuracy: 0.8357 - mae: 0.2355 - mse: 0.1172 - auc_8: 0.9064\n",
      "Epoch 6/10\n",
      "798/798 [==============================] - 4s 6ms/step - loss: 0.3672 - accuracy: 0.8384 - mae: 0.2326 - mse: 0.1155 - auc_8: 0.9088\n",
      "Epoch 7/10\n",
      "798/798 [==============================] - 4s 6ms/step - loss: 0.3618 - accuracy: 0.8413 - mae: 0.2282 - mse: 0.1136 - auc_8: 0.9117\n",
      "Epoch 8/10\n",
      "798/798 [==============================] - 4s 6ms/step - loss: 0.3578 - accuracy: 0.8422 - mae: 0.2261 - mse: 0.1125 - auc_8: 0.9138\n",
      "Epoch 9/10\n",
      "798/798 [==============================] - 4s 6ms/step - loss: 0.3518 - accuracy: 0.8451 - mae: 0.2220 - mse: 0.1106 - auc_8: 0.9170\n",
      "Epoch 10/10\n",
      "798/798 [==============================] - 4s 6ms/step - loss: 0.3481 - accuracy: 0.8473 - mae: 0.2198 - mse: 0.1093 - auc_8: 0.9186\n",
      "84\n",
      "84\n",
      "84\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "82\n",
      "82\n",
      "Epoch 1/10\n",
      "798/798 [==============================] - 4s 6ms/step - loss: 0.3437 - accuracy: 0.8504 - mae: 0.2162 - mse: 0.1076 - auc_8: 0.9207\n",
      "Epoch 2/10\n",
      "798/798 [==============================] - 4s 5ms/step - loss: 0.3404 - accuracy: 0.8520 - mae: 0.2140 - mse: 0.1065 - auc_8: 0.9224\n",
      "Epoch 3/10\n",
      "798/798 [==============================] - 4s 5ms/step - loss: 0.3385 - accuracy: 0.8520 - mae: 0.2131 - mse: 0.1060 - auc_8: 0.9232\n",
      "Epoch 4/10\n",
      "798/798 [==============================] - 4s 6ms/step - loss: 0.3314 - accuracy: 0.8544 - mae: 0.2084 - mse: 0.1037 - auc_8: 0.9267\n",
      "Epoch 5/10\n",
      "798/798 [==============================] - 4s 6ms/step - loss: 0.3296 - accuracy: 0.8564 - mae: 0.2071 - mse: 0.1033 - auc_8: 0.9273\n",
      "Epoch 6/10\n",
      "798/798 [==============================] - 4s 5ms/step - loss: 0.3250 - accuracy: 0.8582 - mae: 0.2038 - mse: 0.1016 - auc_8: 0.9296\n",
      "Epoch 7/10\n",
      "798/798 [==============================] - 4s 5ms/step - loss: 0.3201 - accuracy: 0.8609 - mae: 0.2002 - mse: 0.0998 - auc_8: 0.9318\n",
      "Epoch 8/10\n",
      "798/798 [==============================] - 4s 6ms/step - loss: 0.3198 - accuracy: 0.8608 - mae: 0.2001 - mse: 0.0998 - auc_8: 0.9319\n",
      "Epoch 9/10\n",
      "798/798 [==============================] - 4s 5ms/step - loss: 0.3148 - accuracy: 0.8636 - mae: 0.1971 - mse: 0.0981 - auc_8: 0.9340\n",
      "Epoch 10/10\n",
      "798/798 [==============================] - 4s 5ms/step - loss: 0.3127 - accuracy: 0.8646 - mae: 0.1955 - mse: 0.0973 - auc_8: 0.9351\n",
      "86\n",
      "86\n",
      "86\n",
      "85\n",
      "85\n",
      "85\n",
      "85\n",
      "85\n",
      "85\n",
      "Epoch 1/10\n",
      "798/798 [==============================] - 4s 5ms/step - loss: 0.3098 - accuracy: 0.8677 - mae: 0.1935 - mse: 0.0961 - auc_8: 0.9361\n",
      "Epoch 2/10\n",
      "798/798 [==============================] - 4s 6ms/step - loss: 0.3041 - accuracy: 0.8685 - mae: 0.1890 - mse: 0.0943 - auc_8: 0.9386\n",
      "Epoch 3/10\n",
      "798/798 [==============================] - 4s 5ms/step - loss: 0.3043 - accuracy: 0.8691 - mae: 0.1894 - mse: 0.0946 - auc_8: 0.9384\n",
      "Epoch 4/10\n",
      "798/798 [==============================] - 4s 5ms/step - loss: 0.3011 - accuracy: 0.8713 - mae: 0.1874 - mse: 0.0932 - auc_8: 0.9398\n",
      "Epoch 5/10\n",
      "798/798 [==============================] - 4s 6ms/step - loss: 0.2979 - accuracy: 0.8718 - mae: 0.1863 - mse: 0.0927 - auc_8: 0.9410\n",
      "Epoch 6/10\n",
      "798/798 [==============================] - 4s 5ms/step - loss: 0.2953 - accuracy: 0.8727 - mae: 0.1841 - mse: 0.0918 - auc_8: 0.9420\n",
      "Epoch 7/10\n",
      "798/798 [==============================] - 4s 5ms/step - loss: 0.2928 - accuracy: 0.8740 - mae: 0.1820 - mse: 0.0908 - auc_8: 0.9431\n",
      "Epoch 8/10\n",
      "798/798 [==============================] - 4s 5ms/step - loss: 0.2932 - accuracy: 0.8745 - mae: 0.1814 - mse: 0.0906 - auc_8: 0.9429\n",
      "Epoch 9/10\n",
      "798/798 [==============================] - 4s 6ms/step - loss: 0.2888 - accuracy: 0.8758 - mae: 0.1797 - mse: 0.0895 - auc_8: 0.9447\n",
      "Epoch 10/10\n",
      "798/798 [==============================] - 4s 5ms/step - loss: 0.2853 - accuracy: 0.8789 - mae: 0.1772 - mse: 0.0881 - auc_8: 0.9460\n",
      "87\n",
      "87\n",
      "87\n",
      "87\n",
      "87\n",
      "87\n",
      "86\n",
      "86\n",
      "86\n",
      "Epoch 1/10\n",
      "798/798 [==============================] - 4s 6ms/step - loss: 0.2850 - accuracy: 0.8783 - mae: 0.1762 - mse: 0.0877 - auc_8: 0.9461\n",
      "Epoch 2/10\n",
      "798/798 [==============================] - 4s 6ms/step - loss: 0.2855 - accuracy: 0.8793 - mae: 0.1764 - mse: 0.0880 - auc_8: 0.9460\n",
      "Epoch 3/10\n",
      "798/798 [==============================] - 4s 6ms/step - loss: 0.2835 - accuracy: 0.8796 - mae: 0.1760 - mse: 0.0875 - auc_8: 0.9467\n",
      "Epoch 4/10\n",
      "798/798 [==============================] - 4s 6ms/step - loss: 0.2823 - accuracy: 0.8805 - mae: 0.1744 - mse: 0.0870 - auc_8: 0.9471\n",
      "Epoch 5/10\n",
      "798/798 [==============================] - 4s 6ms/step - loss: 0.2808 - accuracy: 0.8792 - mae: 0.1747 - mse: 0.0869 - auc_8: 0.9476\n",
      "Epoch 6/10\n",
      "798/798 [==============================] - 4s 6ms/step - loss: 0.2789 - accuracy: 0.8812 - mae: 0.1723 - mse: 0.0862 - auc_8: 0.9482\n",
      "Epoch 7/10\n",
      "798/798 [==============================] - 4s 6ms/step - loss: 0.2774 - accuracy: 0.8822 - mae: 0.1718 - mse: 0.0857 - auc_8: 0.9489\n",
      "Epoch 8/10\n",
      "798/798 [==============================] - 4s 6ms/step - loss: 0.2750 - accuracy: 0.8817 - mae: 0.1706 - mse: 0.0850 - auc_8: 0.9496\n",
      "Epoch 9/10\n",
      "798/798 [==============================] - 4s 6ms/step - loss: 0.2802 - accuracy: 0.8803 - mae: 0.1742 - mse: 0.0868 - auc_8: 0.9476\n",
      "Epoch 10/10\n",
      "798/798 [==============================] - 4s 6ms/step - loss: 0.2762 - accuracy: 0.8821 - mae: 0.1712 - mse: 0.0854 - auc_8: 0.9492\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "87\n",
      "88\n",
      "87\n",
      "87\n",
      "87\n",
      "Epoch 1/10\n",
      "798/798 [==============================] - 4s 6ms/step - loss: 0.2758 - accuracy: 0.8816 - mae: 0.1711 - mse: 0.0853 - auc_8: 0.9493\n",
      "Epoch 2/10\n",
      "798/798 [==============================] - 4s 5ms/step - loss: 0.2759 - accuracy: 0.8823 - mae: 0.1709 - mse: 0.0851 - auc_8: 0.9494\n",
      "Epoch 3/10\n",
      "798/798 [==============================] - 4s 6ms/step - loss: 0.2739 - accuracy: 0.8834 - mae: 0.1689 - mse: 0.0844 - auc_8: 0.9501\n",
      "Epoch 4/10\n",
      "798/798 [==============================] - 4s 6ms/step - loss: 0.2726 - accuracy: 0.8831 - mae: 0.1688 - mse: 0.0842 - auc_8: 0.9504\n",
      "Epoch 5/10\n",
      "798/798 [==============================] - 4s 5ms/step - loss: 0.2767 - accuracy: 0.8830 - mae: 0.1713 - mse: 0.0853 - auc_8: 0.9490\n",
      "Epoch 6/10\n",
      "798/798 [==============================] - 4s 5ms/step - loss: 0.2758 - accuracy: 0.8833 - mae: 0.1700 - mse: 0.0850 - auc_8: 0.9493\n",
      "Epoch 7/10\n",
      "798/798 [==============================] - 4s 6ms/step - loss: 0.2730 - accuracy: 0.8833 - mae: 0.1689 - mse: 0.0842 - auc_8: 0.9504\n",
      "Epoch 8/10\n",
      "798/798 [==============================] - 4s 5ms/step - loss: 0.2727 - accuracy: 0.8840 - mae: 0.1689 - mse: 0.0841 - auc_8: 0.9506\n",
      "Epoch 9/10\n",
      "798/798 [==============================] - 4s 5ms/step - loss: 0.2700 - accuracy: 0.8848 - mae: 0.1669 - mse: 0.0830 - auc_8: 0.9514\n",
      "Epoch 10/10\n",
      "798/798 [==============================] - 4s 5ms/step - loss: 0.2720 - accuracy: 0.8854 - mae: 0.1677 - mse: 0.0835 - auc_8: 0.9508\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "6\n",
      "244/244 [==============================] - 0s 730us/step - loss: 0.2315 - accuracy: 0.9034 - mae: 0.1461 - mse: 0.0677 - auc_8: 0.8865\n",
      "train positive label: 5305 - train negative label: 64847\n",
      "up and down sampling => train positive label: 37135 - train negative label: 64847\n",
      "Test positive label: 557 - Test negative label: 7237\n",
      "Epoch 1/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.5223 - accuracy: 0.7539 - mae: 0.3432 - mse: 0.1707 - auc_9: 0.8048\n",
      "Epoch 2/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.4724 - accuracy: 0.7856 - mae: 0.3069 - mse: 0.1518 - auc_9: 0.8429\n",
      "Epoch 3/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.4551 - accuracy: 0.7964 - mae: 0.2935 - mse: 0.1452 - auc_9: 0.8552\n",
      "Epoch 4/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.4452 - accuracy: 0.8013 - mae: 0.2863 - mse: 0.1416 - auc_9: 0.8623\n",
      "Epoch 5/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.4357 - accuracy: 0.8046 - mae: 0.2800 - mse: 0.1387 - auc_9: 0.8678\n",
      "Epoch 6/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.4264 - accuracy: 0.8111 - mae: 0.2735 - mse: 0.1352 - auc_9: 0.8743\n",
      "Epoch 7/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.4171 - accuracy: 0.8147 - mae: 0.2667 - mse: 0.1321 - auc_9: 0.8802\n",
      "Epoch 8/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.4098 - accuracy: 0.8172 - mae: 0.2623 - mse: 0.1298 - auc_9: 0.8847\n",
      "Epoch 9/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.4029 - accuracy: 0.8212 - mae: 0.2572 - mse: 0.1274 - auc_9: 0.8891\n",
      "Epoch 10/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.3962 - accuracy: 0.8228 - mae: 0.2530 - mse: 0.1255 - auc_9: 0.8927\n",
      "82\n",
      "81\n",
      "81\n",
      "81\n",
      "80\n",
      "80\n",
      "79\n",
      "78\n",
      "75\n",
      "Epoch 1/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.3887 - accuracy: 0.8278 - mae: 0.2478 - mse: 0.1229 - auc_9: 0.8972\n",
      "Epoch 2/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.3825 - accuracy: 0.8305 - mae: 0.2437 - mse: 0.1207 - auc_9: 0.9007\n",
      "Epoch 3/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.3767 - accuracy: 0.8331 - mae: 0.2400 - mse: 0.1190 - auc_9: 0.9038\n",
      "Epoch 4/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.3720 - accuracy: 0.8368 - mae: 0.2364 - mse: 0.1171 - auc_9: 0.9065\n",
      "Epoch 5/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.3661 - accuracy: 0.8372 - mae: 0.2327 - mse: 0.1155 - auc_9: 0.9095\n",
      "Epoch 6/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.3596 - accuracy: 0.8407 - mae: 0.2283 - mse: 0.1133 - auc_9: 0.9130\n",
      "Epoch 7/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.3538 - accuracy: 0.8438 - mae: 0.2242 - mse: 0.1113 - auc_9: 0.9158\n",
      "Epoch 8/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.3504 - accuracy: 0.8468 - mae: 0.2218 - mse: 0.1100 - auc_9: 0.9177\n",
      "Epoch 9/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.3436 - accuracy: 0.8485 - mae: 0.2174 - mse: 0.1080 - auc_9: 0.9208\n",
      "Epoch 10/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.3411 - accuracy: 0.8507 - mae: 0.2153 - mse: 0.1069 - auc_9: 0.9222\n",
      "84\n",
      "84\n",
      "84\n",
      "84\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "82\n",
      "Epoch 1/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.3372 - accuracy: 0.8520 - mae: 0.2129 - mse: 0.1058 - auc_9: 0.9239\n",
      "Epoch 2/10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "797/797 [==============================] - 4s 5ms/step - loss: 0.3311 - accuracy: 0.8559 - mae: 0.2084 - mse: 0.1035 - auc_9: 0.9270\n",
      "Epoch 3/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.3288 - accuracy: 0.8562 - mae: 0.2076 - mse: 0.1030 - auc_9: 0.9279\n",
      "Epoch 4/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.3235 - accuracy: 0.8582 - mae: 0.2039 - mse: 0.1014 - auc_9: 0.9301\n",
      "Epoch 5/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.3190 - accuracy: 0.8609 - mae: 0.2007 - mse: 0.0997 - auc_9: 0.9321\n",
      "Epoch 6/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.3169 - accuracy: 0.8620 - mae: 0.1989 - mse: 0.0990 - auc_9: 0.9331\n",
      "Epoch 7/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.3145 - accuracy: 0.8632 - mae: 0.1972 - mse: 0.0980 - auc_9: 0.9343\n",
      "Epoch 8/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.3087 - accuracy: 0.8660 - mae: 0.1937 - mse: 0.0963 - auc_9: 0.9366\n",
      "Epoch 9/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.3057 - accuracy: 0.8668 - mae: 0.1915 - mse: 0.0954 - auc_9: 0.9378\n",
      "Epoch 10/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.3035 - accuracy: 0.8685 - mae: 0.1902 - mse: 0.0946 - auc_9: 0.9387\n",
      "86\n",
      "86\n",
      "86\n",
      "86\n",
      "86\n",
      "85\n",
      "85\n",
      "85\n",
      "85\n",
      "Epoch 1/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.3016 - accuracy: 0.8686 - mae: 0.1894 - mse: 0.0941 - auc_9: 0.9396\n",
      "Epoch 2/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.2983 - accuracy: 0.8702 - mae: 0.1864 - mse: 0.0928 - auc_9: 0.9408\n",
      "Epoch 3/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.2953 - accuracy: 0.8737 - mae: 0.1842 - mse: 0.0917 - auc_9: 0.9421\n",
      "Epoch 4/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.2939 - accuracy: 0.8738 - mae: 0.1836 - mse: 0.0911 - auc_9: 0.9427\n",
      "Epoch 5/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.2912 - accuracy: 0.8748 - mae: 0.1815 - mse: 0.0904 - auc_9: 0.9437\n",
      "Epoch 6/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.2905 - accuracy: 0.8744 - mae: 0.1816 - mse: 0.0903 - auc_9: 0.9439\n",
      "Epoch 7/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.2888 - accuracy: 0.8758 - mae: 0.1806 - mse: 0.0898 - auc_9: 0.9445\n",
      "Epoch 8/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.2868 - accuracy: 0.8769 - mae: 0.1784 - mse: 0.0888 - auc_9: 0.9455\n",
      "Epoch 9/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.2887 - accuracy: 0.8751 - mae: 0.1800 - mse: 0.0897 - auc_9: 0.9446\n",
      "Epoch 10/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.2817 - accuracy: 0.8780 - mae: 0.1761 - mse: 0.0875 - auc_9: 0.9472\n",
      "87\n",
      "87\n",
      "87\n",
      "87\n",
      "87\n",
      "87\n",
      "87\n",
      "87\n",
      "86\n",
      "Epoch 1/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.2835 - accuracy: 0.8773 - mae: 0.1768 - mse: 0.0880 - auc_9: 0.9467\n",
      "Epoch 2/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.2818 - accuracy: 0.8785 - mae: 0.1758 - mse: 0.0874 - auc_9: 0.9472\n",
      "Epoch 3/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.2802 - accuracy: 0.8802 - mae: 0.1744 - mse: 0.0868 - auc_9: 0.9479\n",
      "Epoch 4/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.2780 - accuracy: 0.8806 - mae: 0.1724 - mse: 0.0859 - auc_9: 0.9487\n",
      "Epoch 5/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.2758 - accuracy: 0.8806 - mae: 0.1717 - mse: 0.0855 - auc_9: 0.9495\n",
      "Epoch 6/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.2760 - accuracy: 0.8809 - mae: 0.1719 - mse: 0.0856 - auc_9: 0.9493\n",
      "Epoch 7/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.2799 - accuracy: 0.8794 - mae: 0.1744 - mse: 0.0868 - auc_9: 0.9480\n",
      "Epoch 8/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.2780 - accuracy: 0.8797 - mae: 0.1732 - mse: 0.0863 - auc_9: 0.9486\n",
      "Epoch 9/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.2756 - accuracy: 0.8815 - mae: 0.1712 - mse: 0.0852 - auc_9: 0.9496\n",
      "Epoch 10/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.2728 - accuracy: 0.8823 - mae: 0.1694 - mse: 0.0845 - auc_9: 0.9505\n",
      "88\n",
      "87\n",
      "87\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "87\n",
      "87\n",
      "Epoch 1/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.2736 - accuracy: 0.8826 - mae: 0.1698 - mse: 0.0846 - auc_9: 0.9504\n",
      "Epoch 2/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.2711 - accuracy: 0.8834 - mae: 0.1687 - mse: 0.0839 - auc_9: 0.9511\n",
      "Epoch 3/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.2684 - accuracy: 0.8841 - mae: 0.1665 - mse: 0.0829 - auc_9: 0.9521\n",
      "Epoch 4/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.2686 - accuracy: 0.8849 - mae: 0.1666 - mse: 0.0830 - auc_9: 0.9521\n",
      "Epoch 5/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.2656 - accuracy: 0.8861 - mae: 0.1648 - mse: 0.0821 - auc_9: 0.9530\n",
      "Epoch 6/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.2662 - accuracy: 0.8847 - mae: 0.1656 - mse: 0.0824 - auc_9: 0.9528\n",
      "Epoch 7/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.2621 - accuracy: 0.8871 - mae: 0.1629 - mse: 0.0812 - auc_9: 0.9543\n",
      "Epoch 8/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.2612 - accuracy: 0.8879 - mae: 0.1618 - mse: 0.0807 - auc_9: 0.9546\n",
      "Epoch 9/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.2583 - accuracy: 0.8890 - mae: 0.1602 - mse: 0.0799 - auc_9: 0.9556\n",
      "Epoch 10/10\n",
      "797/797 [==============================] - 4s 5ms/step - loss: 0.2613 - accuracy: 0.8873 - mae: 0.1620 - mse: 0.0808 - auc_9: 0.9545\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "88\n",
      "6\n",
      "244/244 [==============================] - 0s 669us/step - loss: 0.2407 - accuracy: 0.8920 - mae: 0.1465 - mse: 0.0718 - auc_9: 0.8917\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import KFold\n",
    "\n",
    "Epoch_S = 10\n",
    "\n",
    "def evaluate_model(df, k = 10 , shuffle = False):\n",
    "    \n",
    "    result =[] \n",
    "    s = 0\n",
    "\n",
    "\n",
    "    kf = KFold(n_splits=10, shuffle= shuffle, random_state=None)\n",
    "    \n",
    "    for train_index, test_index in kf.split(df):\n",
    "\n",
    "        train_ds = [df[index] for index in train_index] \n",
    "        \n",
    "        valid_ds = [df[index] for index in test_index]\n",
    "        \n",
    "        label_pos , label_neg, _ , _ = count_lablel(train_ds)\n",
    "        print(f'train positive label: {label_pos} - train negative label: {label_neg}')\n",
    "        \n",
    "        train_ds = up_and_down_Samplenig(train_ds, scale_downsampling = 0.5)\n",
    "        \n",
    "        label_pos , label_neg , _ , _ = count_lablel(train_ds)\n",
    "        print(f'up and down sampling => train positive label: {label_pos} - train negative label: {label_neg}')\n",
    "\n",
    "        label_pos , label_neg, _ , _ = count_lablel(valid_ds)\n",
    "        print(f'Test positive label: {label_pos} - Test negative label: {label_neg}')\n",
    "\n",
    "        l_train = []\n",
    "        r_train = []\n",
    "        lbls_train = []\n",
    "        l_valid = []\n",
    "        r_valid = []\n",
    "        lbls_valid = []\n",
    "\n",
    "        for i , data in enumerate(train_ds):\n",
    "            embbed_drug, onehot_task, embbed_task, lbl, task_number, task_name = data\n",
    "            l_train.append(embbed_drug[0])\n",
    "            r_train.append(embbed_task)\n",
    "            lbls_train.append(lbl.tolist())\n",
    "        \n",
    "        for i , data in enumerate(valid_ds):\n",
    "            embbed_drug, onehot_task, embbed_task, lbl, task_number, task_name = data\n",
    "            l_valid.append(embbed_drug[0])\n",
    "            r_valid.append(embbed_task)\n",
    "            lbls_valid.append(lbl.tolist())\n",
    "\n",
    "        l_train = np.array(l_train).reshape(-1,512,1)\n",
    "        r_train = np.array(r_train).reshape(-1,512,1)\n",
    "        lbls_train = np.array(lbls_train)\n",
    "\n",
    "        l_valid = np.array(l_valid).reshape(-1,512,1)\n",
    "        r_valid = np.array(r_valid).reshape(-1,512,1)\n",
    "        lbls_valid = np.array(lbls_valid)\n",
    "\n",
    "        # create neural network model\n",
    "        siamese_net = siamese_model_attentiveFp_tox21()\n",
    "        \n",
    "        history = History()\n",
    "        P = siamese_net.fit([l_train, r_train], lbls_train, epochs = Epoch_S, batch_size = 128, callbacks=[history])\n",
    "\n",
    "        for j in range(100):\n",
    "            C=1\n",
    "            Before = int(P.history['accuracy'][-1]*100)\n",
    "            for i in range(2,Epoch_S+1):\n",
    "                if  int(P.history['accuracy'][-i]*100) == Before:\n",
    "                    C=C+1\n",
    "                else:\n",
    "                    C=1\n",
    "                Before=int(P.history['accuracy'][-i]*100)\n",
    "                print(Before)\n",
    "            if C==Epoch_S:\n",
    "                break\n",
    "            P = siamese_net.fit([l_train, r_train], lbls_train, epochs = Epoch_S, batch_size = 128, callbacks=[history])\n",
    "        print(j+1)\n",
    "        \n",
    "        score  = siamese_net.evaluate([l_valid,r_valid], lbls_valid, verbose=1)\n",
    "        a = (score[1],score[4])\n",
    "        result.append(a)\n",
    "        \n",
    "        if score[4] > s :\n",
    "            best_model = siamese_net\n",
    "            s = score[4]\n",
    "            print(\"Save_model!!\")\n",
    "    \n",
    "    return result , best_model\n",
    "\n",
    "\n",
    "scores, best_model = evaluate_model(data_ds, 10, True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "#### Dropout = 0.3 and downsampling = 0.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0.9014753103256226, 0.8940294981002808),\n",
       " (0.909172534942627, 0.884463906288147),\n",
       " (0.9008338451385498, 0.8929635882377625),\n",
       " (0.8828736543655396, 0.8922119140625),\n",
       " (0.8921103477478027, 0.8960561752319336),\n",
       " (0.8880051374435425, 0.8868852853775024),\n",
       " (0.8727226257324219, 0.8965607285499573),\n",
       " (0.8911983370780945, 0.8897591829299927),\n",
       " (0.8881190419197083, 0.8835336565971375),\n",
       " (0.9008211493492126, 0.8929146528244019)]"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy= 0.8927331984043121 AUC= 0.8909378588199616 STD_AUC= 0.004376682209221188\n"
     ]
    }
   ],
   "source": [
    "acc = []\n",
    "auc = []\n",
    "for i in scores:\n",
    "    acc.append(i[0])\n",
    "    auc.append(i[1])\n",
    "\n",
    "print(f'accuracy= {np.mean(acc)} AUC= {np.mean(auc)} STD_AUC= {np.std(auc)}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "#### Dropout = 0.2 and downsampling = 0.5 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0.9033995866775513, 0.8856201767921448),\n",
       " (0.8992944359779358, 0.8737469911575317),\n",
       " (0.9026299118995667, 0.891227126121521),\n",
       " (0.9049390554428101, 0.8905489444732666),\n",
       " (0.9068633913993835, 0.8824583888053894),\n",
       " (0.9131494760513306, 0.8909887075424194),\n",
       " (0.9098024368286133, 0.8867385387420654),\n",
       " (0.9110854268074036, 0.8700131773948669),\n",
       " (0.9018476009368896, 0.888106644153595),\n",
       " (0.9210931658744812, 0.895555853843689)]"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy= 0.9074104487895965 AUC= 0.885500454902649 STD_AUC= 0.007649230178142343\n"
     ]
    }
   ],
   "source": [
    "acc = []\n",
    "auc = []\n",
    "for i in scores:\n",
    "    acc.append(i[0])\n",
    "    auc.append(i[1])\n",
    "\n",
    "print(f'accuracy= {np.mean(acc)} AUC= {np.mean(auc)} STD_AUC= {np.std(auc)}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "id": "_dyrk9nGcM81"
   },
   "source": [
    "# Classification with BioAct-Het and Canonical GCN "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "hidden": true,
    "id": "UePCH8kkoBE5",
    "outputId": "9a7ab5d3-9594-44f5-bddb-d47a6c996228"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading GCN_canonical_Tox21_pre_trained.pth from https://data.dgl.ai/dgllife/pre_trained/gcn_canonical_tox21.pth...\n",
      "Pretrained model loaded\n"
     ]
    }
   ],
   "source": [
    "model_GCN = 'GCN_canonical_Tox21'\n",
    "gcn_model = get_tox21_model(model_GCN)\n",
    "gcn_model.eval()\n",
    "gcn_model = gcn_model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "hidden": true,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/7265\n",
      "Processing molecule 2000/7265\n",
      "Processing molecule 3000/7265\n",
      "Processing molecule 4000/7265\n",
      "Processing molecule 5000/7265\n",
      "Processing molecule 6000/7265\n",
      "Processing molecule 7000/7265\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/6758\n",
      "Processing molecule 2000/6758\n",
      "Processing molecule 3000/6758\n",
      "Processing molecule 4000/6758\n",
      "Processing molecule 5000/6758\n",
      "Processing molecule 6000/6758\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/6549\n",
      "Processing molecule 2000/6549\n",
      "Processing molecule 3000/6549\n",
      "Processing molecule 4000/6549\n",
      "Processing molecule 5000/6549\n",
      "Processing molecule 6000/6549\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/5821\n",
      "Processing molecule 2000/5821\n",
      "Processing molecule 3000/5821\n",
      "Processing molecule 4000/5821\n",
      "Processing molecule 5000/5821\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/6193\n",
      "Processing molecule 2000/6193\n",
      "Processing molecule 3000/6193\n",
      "Processing molecule 4000/6193\n",
      "Processing molecule 5000/6193\n",
      "Processing molecule 6000/6193\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/6955\n",
      "Processing molecule 2000/6955\n",
      "Processing molecule 3000/6955\n",
      "Processing molecule 4000/6955\n",
      "Processing molecule 5000/6955\n",
      "Processing molecule 6000/6955\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/6450\n",
      "Processing molecule 2000/6450\n",
      "Processing molecule 3000/6450\n",
      "Processing molecule 4000/6450\n",
      "Processing molecule 5000/6450\n",
      "Processing molecule 6000/6450\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/5832\n",
      "Processing molecule 2000/5832\n",
      "Processing molecule 3000/5832\n",
      "Processing molecule 4000/5832\n",
      "Processing molecule 5000/5832\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/7072\n",
      "Processing molecule 2000/7072\n",
      "Processing molecule 3000/7072\n",
      "Processing molecule 4000/7072\n",
      "Processing molecule 5000/7072\n",
      "Processing molecule 6000/7072\n",
      "Processing molecule 7000/7072\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/6467\n",
      "Processing molecule 2000/6467\n",
      "Processing molecule 3000/6467\n",
      "Processing molecule 4000/6467\n",
      "Processing molecule 5000/6467\n",
      "Processing molecule 6000/6467\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/5810\n",
      "Processing molecule 2000/5810\n",
      "Processing molecule 3000/5810\n",
      "Processing molecule 4000/5810\n",
      "Processing molecule 5000/5810\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/6774\n",
      "Processing molecule 2000/6774\n",
      "Processing molecule 3000/6774\n",
      "Processing molecule 4000/6774\n",
      "Processing molecule 5000/6774\n",
      "Processing molecule 6000/6774\n",
      "Data created!!\n"
     ]
    }
   ],
   "source": [
    "data_ds = []\n",
    "for i, task in  enumerate(tox21_tasks):\n",
    "    a = df[['smiles' , task]]\n",
    "    a = a.dropna()\n",
    "    ds =  DATASET(a, smiles_to_bigraph, CanonicalAtomFeaturizer(), cache_file_path = cache_path)\n",
    "    data = create_dataset_with_gcn(ds, embed_class_tox21, gcn_model, tox21_tasks, i)\n",
    "    for d in data:\n",
    "        data_ds.append(d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {
     "background_save": true,
     "base_uri": "https://localhost:8080/"
    },
    "collapsed": true,
    "hidden": true,
    "id": "rEpMXuj7ndA7",
    "outputId": "7d6726e0-4792-4d06-953e-648851f6117a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train positive label: 5270 - train negative label: 64881\n",
      "up and down sampling => train positive label: 36890 - train negative label: 64881\n",
      "Test positive label: 592 - Test negative label: 7203\n",
      "Epoch 1/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.5040 - accuracy: 0.7638 - mae: 0.3318 - mse: 0.1649 - auc_10: 0.8151\n",
      "Epoch 2/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4639 - accuracy: 0.7893 - mae: 0.3007 - mse: 0.1491 - auc_10: 0.8459\n",
      "Epoch 3/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4456 - accuracy: 0.7985 - mae: 0.2874 - mse: 0.1426 - auc_10: 0.8590\n",
      "Epoch 4/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4316 - accuracy: 0.8052 - mae: 0.2772 - mse: 0.1377 - auc_10: 0.8688\n",
      "Epoch 5/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4211 - accuracy: 0.8117 - mae: 0.2699 - mse: 0.1340 - auc_10: 0.8756\n",
      "Epoch 6/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4136 - accuracy: 0.8154 - mae: 0.2645 - mse: 0.1315 - auc_10: 0.8804\n",
      "Epoch 7/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4060 - accuracy: 0.8192 - mae: 0.2584 - mse: 0.1287 - auc_10: 0.8854\n",
      "Epoch 8/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3980 - accuracy: 0.8237 - mae: 0.2533 - mse: 0.1260 - auc_10: 0.8900\n",
      "Epoch 9/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3910 - accuracy: 0.8263 - mae: 0.2483 - mse: 0.1236 - auc_10: 0.8945\n",
      "Epoch 10/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3847 - accuracy: 0.8295 - mae: 0.2442 - mse: 0.1216 - auc_10: 0.8980\n",
      "82\n",
      "82\n",
      "81\n",
      "81\n",
      "81\n",
      "80\n",
      "79\n",
      "78\n",
      "76\n",
      "Epoch 1/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3792 - accuracy: 0.8330 - mae: 0.2403 - mse: 0.1196 - auc_10: 0.9012\n",
      "Epoch 2/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3736 - accuracy: 0.8345 - mae: 0.2369 - mse: 0.1177 - auc_10: 0.9045\n",
      "Epoch 3/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3740 - accuracy: 0.8345 - mae: 0.2367 - mse: 0.1181 - auc_10: 0.9042\n",
      "Epoch 4/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3706 - accuracy: 0.8352 - mae: 0.2350 - mse: 0.1171 - auc_10: 0.9061\n",
      "Epoch 5/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3694 - accuracy: 0.8364 - mae: 0.2341 - mse: 0.1166 - auc_10: 0.9069\n",
      "Epoch 6/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3663 - accuracy: 0.8367 - mae: 0.2327 - mse: 0.1159 - auc_10: 0.9084\n",
      "Epoch 7/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3647 - accuracy: 0.8397 - mae: 0.2307 - mse: 0.1150 - auc_10: 0.9093\n",
      "Epoch 8/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3624 - accuracy: 0.8388 - mae: 0.2293 - mse: 0.1146 - auc_10: 0.9105\n",
      "Epoch 9/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3595 - accuracy: 0.8405 - mae: 0.2278 - mse: 0.1136 - auc_10: 0.9119\n",
      "Epoch 10/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3581 - accuracy: 0.8416 - mae: 0.2265 - mse: 0.1130 - auc_10: 0.9128\n",
      "84\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "Epoch 1/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3603 - accuracy: 0.8397 - mae: 0.2287 - mse: 0.1140 - auc_10: 0.9116\n",
      "Epoch 2/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3625 - accuracy: 0.8379 - mae: 0.2301 - mse: 0.1149 - auc_10: 0.9104\n",
      "Epoch 3/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3658 - accuracy: 0.8359 - mae: 0.2330 - mse: 0.1162 - auc_10: 0.9086\n",
      "Epoch 4/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3647 - accuracy: 0.8356 - mae: 0.2323 - mse: 0.1158 - auc_10: 0.9093\n",
      "Epoch 5/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3682 - accuracy: 0.8332 - mae: 0.2348 - mse: 0.1172 - auc_10: 0.9074\n",
      "Epoch 6/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3710 - accuracy: 0.8322 - mae: 0.2370 - mse: 0.1180 - auc_10: 0.9060\n",
      "Epoch 7/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3738 - accuracy: 0.8326 - mae: 0.2381 - mse: 0.1189 - auc_10: 0.9045\n",
      "Epoch 8/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3703 - accuracy: 0.8336 - mae: 0.2362 - mse: 0.1179 - auc_10: 0.9063\n",
      "Epoch 9/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3675 - accuracy: 0.8344 - mae: 0.2342 - mse: 0.1170 - auc_10: 0.9077\n",
      "Epoch 10/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3721 - accuracy: 0.8316 - mae: 0.2374 - mse: 0.1184 - auc_10: 0.9053\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "3\n",
      "244/244 [==============================] - 0s 636us/step - loss: 0.3079 - accuracy: 0.8568 - mae: 0.2120 - mse: 0.0954 - auc_10: 0.8622\n",
      "train positive label: 5296 - train negative label: 64855\n",
      "up and down sampling => train positive label: 37072 - train negative label: 64855\n",
      "Test positive label: 566 - Test negative label: 7229\n",
      "Epoch 1/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.5103 - accuracy: 0.7573 - mae: 0.3374 - mse: 0.1676 - auc_11: 0.8096\n",
      "Epoch 2/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.4643 - accuracy: 0.7864 - mae: 0.3015 - mse: 0.1498 - auc_11: 0.8448\n",
      "Epoch 3/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.4478 - accuracy: 0.7959 - mae: 0.2891 - mse: 0.1437 - auc_11: 0.8571\n",
      "Epoch 4/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.4365 - accuracy: 0.8018 - mae: 0.2808 - mse: 0.1396 - auc_11: 0.8652\n",
      "Epoch 5/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.4258 - accuracy: 0.8094 - mae: 0.2727 - mse: 0.1357 - auc_11: 0.8722\n",
      "Epoch 6/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.4169 - accuracy: 0.8136 - mae: 0.2661 - mse: 0.1325 - auc_11: 0.8782\n",
      "Epoch 7/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.4106 - accuracy: 0.8167 - mae: 0.2618 - mse: 0.1304 - auc_11: 0.8821\n",
      "Epoch 8/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.4006 - accuracy: 0.8226 - mae: 0.2548 - mse: 0.1267 - auc_11: 0.8885\n",
      "Epoch 9/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3940 - accuracy: 0.8244 - mae: 0.2505 - mse: 0.1248 - auc_11: 0.8922\n",
      "Epoch 10/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3896 - accuracy: 0.8270 - mae: 0.2471 - mse: 0.1231 - auc_11: 0.8951\n",
      "82\n",
      "82\n",
      "81\n",
      "81\n",
      "80\n",
      "80\n",
      "79\n",
      "78\n",
      "75\n",
      "Epoch 1/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3852 - accuracy: 0.8293 - mae: 0.2441 - mse: 0.1217 - auc_11: 0.8976\n",
      "Epoch 2/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3782 - accuracy: 0.8323 - mae: 0.2397 - mse: 0.1193 - auc_11: 0.9017\n",
      "Epoch 3/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3751 - accuracy: 0.8347 - mae: 0.2371 - mse: 0.1182 - auc_11: 0.9035\n",
      "Epoch 4/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3736 - accuracy: 0.8345 - mae: 0.2365 - mse: 0.1180 - auc_11: 0.9044\n",
      "Epoch 5/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3723 - accuracy: 0.8349 - mae: 0.2358 - mse: 0.1175 - auc_11: 0.9051\n",
      "Epoch 6/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3741 - accuracy: 0.8346 - mae: 0.2371 - mse: 0.1181 - auc_11: 0.9042\n",
      "Epoch 7/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3739 - accuracy: 0.8341 - mae: 0.2368 - mse: 0.1182 - auc_11: 0.9041\n",
      "Epoch 8/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3705 - accuracy: 0.8354 - mae: 0.2346 - mse: 0.1171 - auc_11: 0.9061\n",
      "Epoch 9/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3763 - accuracy: 0.8325 - mae: 0.2388 - mse: 0.1192 - auc_11: 0.9031\n",
      "Epoch 10/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3768 - accuracy: 0.8315 - mae: 0.2394 - mse: 0.1195 - auc_11: 0.9030\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "82\n",
      "Epoch 1/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3755 - accuracy: 0.8320 - mae: 0.2388 - mse: 0.1192 - auc_11: 0.9037\n",
      "Epoch 2/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3737 - accuracy: 0.8339 - mae: 0.2371 - mse: 0.1183 - auc_11: 0.9048\n",
      "Epoch 3/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3718 - accuracy: 0.8337 - mae: 0.2362 - mse: 0.1180 - auc_11: 0.9058\n",
      "Epoch 4/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3759 - accuracy: 0.8312 - mae: 0.2393 - mse: 0.1195 - auc_11: 0.9036\n",
      "Epoch 5/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3791 - accuracy: 0.8289 - mae: 0.2420 - mse: 0.1208 - auc_11: 0.9018\n",
      "Epoch 6/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3780 - accuracy: 0.8290 - mae: 0.2414 - mse: 0.1205 - auc_11: 0.9023\n",
      "Epoch 7/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3743 - accuracy: 0.8320 - mae: 0.2386 - mse: 0.1190 - auc_11: 0.9046\n",
      "Epoch 8/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3702 - accuracy: 0.8336 - mae: 0.2359 - mse: 0.1178 - auc_11: 0.9066\n",
      "Epoch 9/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3744 - accuracy: 0.8308 - mae: 0.2391 - mse: 0.1194 - auc_11: 0.9043\n",
      "Epoch 10/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3832 - accuracy: 0.8254 - mae: 0.2451 - mse: 0.1225 - auc_11: 0.8994\n",
      "83\n",
      "83\n",
      "83\n",
      "82\n",
      "82\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "Epoch 1/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3812 - accuracy: 0.8264 - mae: 0.2444 - mse: 0.1219 - auc_11: 0.9004\n",
      "Epoch 2/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3792 - accuracy: 0.8261 - mae: 0.2428 - mse: 0.1213 - auc_11: 0.9014\n",
      "Epoch 3/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3776 - accuracy: 0.8273 - mae: 0.2424 - mse: 0.1211 - auc_11: 0.9022\n",
      "Epoch 4/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3763 - accuracy: 0.8277 - mae: 0.2416 - mse: 0.1208 - auc_11: 0.9027\n",
      "Epoch 5/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3727 - accuracy: 0.8294 - mae: 0.2391 - mse: 0.1193 - auc_11: 0.9048\n",
      "Epoch 6/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3713 - accuracy: 0.8300 - mae: 0.2377 - mse: 0.1188 - auc_11: 0.9055\n",
      "Epoch 7/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3704 - accuracy: 0.8301 - mae: 0.2375 - mse: 0.1185 - auc_11: 0.9061\n",
      "Epoch 8/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3678 - accuracy: 0.8314 - mae: 0.2356 - mse: 0.1177 - auc_11: 0.9074\n",
      "Epoch 9/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3832 - accuracy: 0.8243 - mae: 0.2458 - mse: 0.1229 - auc_11: 0.8990\n",
      "Epoch 10/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3841 - accuracy: 0.8232 - mae: 0.2463 - mse: 0.1232 - auc_11: 0.8985\n",
      "82\n",
      "83\n",
      "83\n",
      "82\n",
      "82\n",
      "82\n",
      "82\n",
      "82\n",
      "82\n",
      "Epoch 1/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3838 - accuracy: 0.8245 - mae: 0.2468 - mse: 0.1231 - auc_11: 0.8987\n",
      "Epoch 2/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3812 - accuracy: 0.8245 - mae: 0.2450 - mse: 0.1225 - auc_11: 0.9000\n",
      "Epoch 3/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3821 - accuracy: 0.8248 - mae: 0.2455 - mse: 0.1226 - auc_11: 0.8994\n",
      "Epoch 4/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3791 - accuracy: 0.8256 - mae: 0.2437 - mse: 0.1217 - auc_11: 0.9012\n",
      "Epoch 5/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3766 - accuracy: 0.8272 - mae: 0.2418 - mse: 0.1207 - auc_11: 0.9025\n",
      "Epoch 6/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3756 - accuracy: 0.8277 - mae: 0.2415 - mse: 0.1208 - auc_11: 0.9027\n",
      "Epoch 7/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3747 - accuracy: 0.8278 - mae: 0.2410 - mse: 0.1202 - auc_11: 0.9035\n",
      "Epoch 8/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3742 - accuracy: 0.8271 - mae: 0.2409 - mse: 0.1204 - auc_11: 0.9035\n",
      "Epoch 9/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3734 - accuracy: 0.8284 - mae: 0.2395 - mse: 0.1197 - auc_11: 0.9042\n",
      "Epoch 10/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3708 - accuracy: 0.8297 - mae: 0.2380 - mse: 0.1190 - auc_11: 0.9054\n",
      "82\n",
      "82\n",
      "82\n",
      "82\n",
      "82\n",
      "82\n",
      "82\n",
      "82\n",
      "82\n",
      "5\n",
      "244/244 [==============================] - 0s 660us/step - loss: 0.3125 - accuracy: 0.8557 - mae: 0.2101 - mse: 0.0981 - auc_11: 0.8610\n",
      "train positive label: 5267 - train negative label: 64884\n",
      "up and down sampling => train positive label: 36869 - train negative label: 64884\n",
      "Test positive label: 595 - Test negative label: 7200\n",
      "Epoch 1/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.5091 - accuracy: 0.7589 - mae: 0.3358 - mse: 0.1670 - auc_12: 0.8101\n",
      "Epoch 2/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.4628 - accuracy: 0.7889 - mae: 0.2998 - mse: 0.1489 - auc_12: 0.8462\n",
      "Epoch 3/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.4467 - accuracy: 0.7970 - mae: 0.2881 - mse: 0.1431 - auc_12: 0.8577\n",
      "Epoch 4/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.4336 - accuracy: 0.8056 - mae: 0.2783 - mse: 0.1381 - auc_12: 0.8670\n",
      "Epoch 5/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.4244 - accuracy: 0.8103 - mae: 0.2713 - mse: 0.1348 - auc_12: 0.8730\n",
      "Epoch 6/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.4152 - accuracy: 0.8149 - mae: 0.2646 - mse: 0.1317 - auc_12: 0.8788\n",
      "Epoch 7/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.4079 - accuracy: 0.8183 - mae: 0.2594 - mse: 0.1291 - auc_12: 0.8836\n",
      "Epoch 8/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.4014 - accuracy: 0.8228 - mae: 0.2546 - mse: 0.1268 - auc_12: 0.8876\n",
      "Epoch 9/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3952 - accuracy: 0.8256 - mae: 0.2503 - mse: 0.1247 - auc_12: 0.8915\n",
      "Epoch 10/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3876 - accuracy: 0.8294 - mae: 0.2454 - mse: 0.1223 - auc_12: 0.8956\n",
      "82\n",
      "82\n",
      "81\n",
      "81\n",
      "81\n",
      "80\n",
      "79\n",
      "78\n",
      "75\n",
      "Epoch 1/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3857 - accuracy: 0.8295 - mae: 0.2442 - mse: 0.1216 - auc_12: 0.8971\n",
      "Epoch 2/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3780 - accuracy: 0.8341 - mae: 0.2390 - mse: 0.1189 - auc_12: 0.9016\n",
      "Epoch 3/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3770 - accuracy: 0.8345 - mae: 0.2380 - mse: 0.1187 - auc_12: 0.9023\n",
      "Epoch 4/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3720 - accuracy: 0.8372 - mae: 0.2347 - mse: 0.1169 - auc_12: 0.9050\n",
      "Epoch 5/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3686 - accuracy: 0.8386 - mae: 0.2328 - mse: 0.1158 - auc_12: 0.9071\n",
      "Epoch 6/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3695 - accuracy: 0.8387 - mae: 0.2327 - mse: 0.1161 - auc_12: 0.9066\n",
      "Epoch 7/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3662 - accuracy: 0.8398 - mae: 0.2308 - mse: 0.1151 - auc_12: 0.9084\n",
      "Epoch 8/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3634 - accuracy: 0.8411 - mae: 0.2294 - mse: 0.1142 - auc_12: 0.9099\n",
      "Epoch 9/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3635 - accuracy: 0.8403 - mae: 0.2294 - mse: 0.1145 - auc_12: 0.9098\n",
      "Epoch 10/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3581 - accuracy: 0.8441 - mae: 0.2256 - mse: 0.1124 - auc_12: 0.9128\n",
      "84\n",
      "84\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "82\n",
      "Epoch 1/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3593 - accuracy: 0.8417 - mae: 0.2264 - mse: 0.1131 - auc_12: 0.9122\n",
      "Epoch 2/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3545 - accuracy: 0.8451 - mae: 0.2235 - mse: 0.1115 - auc_12: 0.9147\n",
      "Epoch 3/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3560 - accuracy: 0.8428 - mae: 0.2251 - mse: 0.1121 - auc_12: 0.9140\n",
      "Epoch 4/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3631 - accuracy: 0.8386 - mae: 0.2299 - mse: 0.1147 - auc_12: 0.9103\n",
      "Epoch 5/10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3664 - accuracy: 0.8375 - mae: 0.2325 - mse: 0.1160 - auc_12: 0.9085\n",
      "Epoch 6/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3696 - accuracy: 0.8360 - mae: 0.2344 - mse: 0.1172 - auc_12: 0.9065\n",
      "Epoch 7/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3682 - accuracy: 0.8358 - mae: 0.2342 - mse: 0.1168 - auc_12: 0.9073\n",
      "Epoch 8/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3645 - accuracy: 0.8378 - mae: 0.2307 - mse: 0.1154 - auc_12: 0.9094\n",
      "Epoch 9/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3637 - accuracy: 0.8376 - mae: 0.2309 - mse: 0.1152 - auc_12: 0.9100\n",
      "Epoch 10/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3630 - accuracy: 0.8382 - mae: 0.2304 - mse: 0.1149 - auc_12: 0.9102\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "84\n",
      "84\n",
      "84\n",
      "Epoch 1/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3643 - accuracy: 0.8368 - mae: 0.2316 - mse: 0.1157 - auc_12: 0.9094\n",
      "Epoch 2/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3642 - accuracy: 0.8354 - mae: 0.2321 - mse: 0.1159 - auc_12: 0.9095\n",
      "Epoch 3/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3686 - accuracy: 0.8341 - mae: 0.2349 - mse: 0.1173 - auc_12: 0.9070\n",
      "Epoch 4/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3674 - accuracy: 0.8340 - mae: 0.2339 - mse: 0.1168 - auc_12: 0.9079\n",
      "Epoch 5/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3622 - accuracy: 0.8367 - mae: 0.2310 - mse: 0.1152 - auc_12: 0.9104\n",
      "Epoch 6/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3620 - accuracy: 0.8378 - mae: 0.2305 - mse: 0.1149 - auc_12: 0.9107\n",
      "Epoch 7/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3623 - accuracy: 0.8368 - mae: 0.2305 - mse: 0.1151 - auc_12: 0.9106\n",
      "Epoch 8/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3582 - accuracy: 0.8380 - mae: 0.2281 - mse: 0.1141 - auc_12: 0.9124\n",
      "Epoch 9/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3575 - accuracy: 0.8396 - mae: 0.2275 - mse: 0.1136 - auc_12: 0.9131\n",
      "Epoch 10/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3614 - accuracy: 0.8371 - mae: 0.2305 - mse: 0.1150 - auc_12: 0.9108\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "4\n",
      "  1/244 [..............................] - ETA: 0s - loss: 0.0928 - accuracy: 1.0000 - mae: 0.0801 - mse: 0.0192 - auc_12: 0.0000e+00WARNING:tensorflow:Callbacks method `on_test_batch_end` is slow compared to the batch time (batch time: 0.0000s vs `on_test_batch_end` time: 0.0010s). Check your callbacks.\n",
      "244/244 [==============================] - 0s 656us/step - loss: 0.2785 - accuracy: 0.8727 - mae: 0.1843 - mse: 0.0866 - auc_12: 0.8781\n",
      "train positive label: 5306 - train negative label: 64845\n",
      "up and down sampling => train positive label: 37142 - train negative label: 64845\n",
      "Test positive label: 556 - Test negative label: 7239\n",
      "Epoch 1/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.5088 - accuracy: 0.7585 - mae: 0.3366 - mse: 0.1671 - auc_13: 0.8099\n",
      "Epoch 2/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.4625 - accuracy: 0.7889 - mae: 0.3001 - mse: 0.1490 - auc_13: 0.8468\n",
      "Epoch 3/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.4461 - accuracy: 0.7941 - mae: 0.2880 - mse: 0.1435 - auc_13: 0.8585\n",
      "Epoch 4/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.4347 - accuracy: 0.8020 - mae: 0.2795 - mse: 0.1390 - auc_13: 0.8667\n",
      "Epoch 5/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.4241 - accuracy: 0.8076 - mae: 0.2714 - mse: 0.1351 - auc_13: 0.8738\n",
      "Epoch 6/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.4147 - accuracy: 0.8143 - mae: 0.2644 - mse: 0.1316 - auc_13: 0.8799\n",
      "Epoch 7/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.4059 - accuracy: 0.8200 - mae: 0.2578 - mse: 0.1284 - auc_13: 0.8852\n",
      "Epoch 8/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.4002 - accuracy: 0.8228 - mae: 0.2536 - mse: 0.1265 - auc_13: 0.8889\n",
      "Epoch 9/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3928 - accuracy: 0.8264 - mae: 0.2492 - mse: 0.1240 - auc_13: 0.8931\n",
      "Epoch 10/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3871 - accuracy: 0.8294 - mae: 0.2451 - mse: 0.1221 - auc_13: 0.8965\n",
      "82\n",
      "82\n",
      "82\n",
      "81\n",
      "80\n",
      "80\n",
      "79\n",
      "78\n",
      "75\n",
      "Epoch 1/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3825 - accuracy: 0.8325 - mae: 0.2416 - mse: 0.1204 - auc_13: 0.8993\n",
      "Epoch 2/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3788 - accuracy: 0.8322 - mae: 0.2398 - mse: 0.1195 - auc_13: 0.9014\n",
      "Epoch 3/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3737 - accuracy: 0.8358 - mae: 0.2359 - mse: 0.1176 - auc_13: 0.9043\n",
      "Epoch 4/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3717 - accuracy: 0.8366 - mae: 0.2345 - mse: 0.1169 - auc_13: 0.9055\n",
      "Epoch 5/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3710 - accuracy: 0.8367 - mae: 0.2341 - mse: 0.1167 - auc_13: 0.9059\n",
      "Epoch 6/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3674 - accuracy: 0.8388 - mae: 0.2317 - mse: 0.1155 - auc_13: 0.9079\n",
      "Epoch 7/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3631 - accuracy: 0.8413 - mae: 0.2289 - mse: 0.1140 - auc_13: 0.9102\n",
      "Epoch 8/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3589 - accuracy: 0.8420 - mae: 0.2261 - mse: 0.1128 - auc_13: 0.9124\n",
      "Epoch 9/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3601 - accuracy: 0.8411 - mae: 0.2273 - mse: 0.1134 - auc_13: 0.9118\n",
      "Epoch 10/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3615 - accuracy: 0.8407 - mae: 0.2280 - mse: 0.1137 - auc_13: 0.9113\n",
      "84\n",
      "84\n",
      "84\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "Epoch 1/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3581 - accuracy: 0.8429 - mae: 0.2263 - mse: 0.1128 - auc_13: 0.9129\n",
      "Epoch 2/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3658 - accuracy: 0.8372 - mae: 0.2315 - mse: 0.1155 - auc_13: 0.9088\n",
      "Epoch 3/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3632 - accuracy: 0.8398 - mae: 0.2291 - mse: 0.1144 - auc_13: 0.9101\n",
      "Epoch 4/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3549 - accuracy: 0.8434 - mae: 0.2240 - mse: 0.1118 - auc_13: 0.9146\n",
      "Epoch 5/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3524 - accuracy: 0.8450 - mae: 0.2221 - mse: 0.1109 - auc_13: 0.9158\n",
      "Epoch 6/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3541 - accuracy: 0.8430 - mae: 0.2242 - mse: 0.1118 - auc_13: 0.9151\n",
      "Epoch 7/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3527 - accuracy: 0.8431 - mae: 0.2226 - mse: 0.1112 - auc_13: 0.9159\n",
      "Epoch 8/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3498 - accuracy: 0.8456 - mae: 0.2210 - mse: 0.1103 - auc_13: 0.9172\n",
      "Epoch 9/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3475 - accuracy: 0.8475 - mae: 0.2192 - mse: 0.1094 - auc_13: 0.9183\n",
      "Epoch 10/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3478 - accuracy: 0.8464 - mae: 0.2195 - mse: 0.1094 - auc_13: 0.9183\n",
      "84\n",
      "84\n",
      "84\n",
      "84\n",
      "84\n",
      "84\n",
      "83\n",
      "83\n",
      "84\n",
      "Epoch 1/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3542 - accuracy: 0.8432 - mae: 0.2237 - mse: 0.1116 - auc_13: 0.9150\n",
      "Epoch 2/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3552 - accuracy: 0.8418 - mae: 0.2247 - mse: 0.1123 - auc_13: 0.9142\n",
      "Epoch 3/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3679 - accuracy: 0.8371 - mae: 0.2331 - mse: 0.1165 - auc_13: 0.9079\n",
      "Epoch 4/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3726 - accuracy: 0.8344 - mae: 0.2366 - mse: 0.1182 - auc_13: 0.9053\n",
      "Epoch 5/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3833 - accuracy: 0.8272 - mae: 0.2449 - mse: 0.1221 - auc_13: 0.8993\n",
      "Epoch 6/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3836 - accuracy: 0.8276 - mae: 0.2448 - mse: 0.1223 - auc_13: 0.8988\n",
      "Epoch 7/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3823 - accuracy: 0.8294 - mae: 0.2435 - mse: 0.1216 - auc_13: 0.8999\n",
      "Epoch 8/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3794 - accuracy: 0.8307 - mae: 0.2418 - mse: 0.1208 - auc_13: 0.9015\n",
      "Epoch 9/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3783 - accuracy: 0.8301 - mae: 0.2419 - mse: 0.1205 - auc_13: 0.9020\n",
      "Epoch 10/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3791 - accuracy: 0.8293 - mae: 0.2416 - mse: 0.1209 - auc_13: 0.9015\n",
      "83\n",
      "83\n",
      "82\n",
      "82\n",
      "82\n",
      "83\n",
      "83\n",
      "84\n",
      "84\n",
      "Epoch 1/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3805 - accuracy: 0.8282 - mae: 0.2430 - mse: 0.1214 - auc_13: 0.9007\n",
      "Epoch 2/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3811 - accuracy: 0.8281 - mae: 0.2436 - mse: 0.1217 - auc_13: 0.9005\n",
      "Epoch 3/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3804 - accuracy: 0.8268 - mae: 0.2437 - mse: 0.1218 - auc_13: 0.9008\n",
      "Epoch 4/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3797 - accuracy: 0.8273 - mae: 0.2435 - mse: 0.1216 - auc_13: 0.9011\n",
      "Epoch 5/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3775 - accuracy: 0.8283 - mae: 0.2420 - mse: 0.1208 - auc_13: 0.9021\n",
      "Epoch 6/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3780 - accuracy: 0.8279 - mae: 0.2421 - mse: 0.1210 - auc_13: 0.9019\n",
      "Epoch 7/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3770 - accuracy: 0.8282 - mae: 0.2421 - mse: 0.1208 - auc_13: 0.9025\n",
      "Epoch 8/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3756 - accuracy: 0.8287 - mae: 0.2409 - mse: 0.1204 - auc_13: 0.9032\n",
      "Epoch 9/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3733 - accuracy: 0.8299 - mae: 0.2397 - mse: 0.1197 - auc_13: 0.9044\n",
      "Epoch 10/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3755 - accuracy: 0.8290 - mae: 0.2406 - mse: 0.1202 - auc_13: 0.9034\n",
      "82\n",
      "82\n",
      "82\n",
      "82\n",
      "82\n",
      "82\n",
      "82\n",
      "82\n",
      "82\n",
      "5\n",
      "244/244 [==============================] - 0s 640us/step - loss: 0.3047 - accuracy: 0.8552 - mae: 0.2077 - mse: 0.0942 - auc_13: 0.8481\n",
      "train positive label: 5245 - train negative label: 64906\n",
      "up and down sampling => train positive label: 36715 - train negative label: 64906\n",
      "Test positive label: 617 - Test negative label: 7178\n",
      "Epoch 1/10\n",
      "794/794 [==============================] - 3s 4ms/step - loss: 0.5107 - accuracy: 0.7557 - mae: 0.3376 - mse: 0.1676 - auc_14: 0.8096\n",
      "Epoch 2/10\n",
      "794/794 [==============================] - 3s 4ms/step - loss: 0.4597 - accuracy: 0.7881 - mae: 0.2985 - mse: 0.1481 - auc_14: 0.8488\n",
      "Epoch 3/10\n",
      "794/794 [==============================] - 3s 4ms/step - loss: 0.4404 - accuracy: 0.8003 - mae: 0.2838 - mse: 0.1410 - auc_14: 0.8619\n",
      "Epoch 4/10\n",
      "794/794 [==============================] - 3s 4ms/step - loss: 0.4275 - accuracy: 0.8076 - mae: 0.2744 - mse: 0.1361 - auc_14: 0.8711\n",
      "Epoch 5/10\n",
      "794/794 [==============================] - 3s 4ms/step - loss: 0.4185 - accuracy: 0.8119 - mae: 0.2679 - mse: 0.1332 - auc_14: 0.8768\n",
      "Epoch 6/10\n",
      "794/794 [==============================] - 3s 4ms/step - loss: 0.4093 - accuracy: 0.8177 - mae: 0.2613 - mse: 0.1298 - auc_14: 0.8826\n",
      "Epoch 7/10\n",
      "794/794 [==============================] - 3s 4ms/step - loss: 0.4004 - accuracy: 0.8219 - mae: 0.2552 - mse: 0.1268 - auc_14: 0.8884\n",
      "Epoch 8/10\n",
      "794/794 [==============================] - 3s 4ms/step - loss: 0.3935 - accuracy: 0.8247 - mae: 0.2504 - mse: 0.1246 - auc_14: 0.8923\n",
      "Epoch 9/10\n",
      "794/794 [==============================] - 3s 4ms/step - loss: 0.3874 - accuracy: 0.8272 - mae: 0.2463 - mse: 0.1224 - auc_14: 0.8962\n",
      "Epoch 10/10\n",
      "794/794 [==============================] - 3s 4ms/step - loss: 0.3817 - accuracy: 0.8303 - mae: 0.2424 - mse: 0.1207 - auc_14: 0.8993\n",
      "82\n",
      "82\n",
      "82\n",
      "81\n",
      "81\n",
      "80\n",
      "80\n",
      "78\n",
      "75\n",
      "Epoch 1/10\n",
      "794/794 [==============================] - 3s 4ms/step - loss: 0.3771 - accuracy: 0.8327 - mae: 0.2392 - mse: 0.1189 - auc_14: 0.9021\n",
      "Epoch 2/10\n",
      "794/794 [==============================] - 3s 4ms/step - loss: 0.3730 - accuracy: 0.8356 - mae: 0.2362 - mse: 0.1176 - auc_14: 0.9044\n",
      "Epoch 3/10\n",
      "794/794 [==============================] - 3s 4ms/step - loss: 0.3699 - accuracy: 0.8364 - mae: 0.2341 - mse: 0.1166 - auc_14: 0.9063\n",
      "Epoch 4/10\n",
      "794/794 [==============================] - 3s 4ms/step - loss: 0.3630 - accuracy: 0.8399 - mae: 0.2295 - mse: 0.1143 - auc_14: 0.9100\n",
      "Epoch 5/10\n",
      "794/794 [==============================] - 3s 4ms/step - loss: 0.3614 - accuracy: 0.8400 - mae: 0.2286 - mse: 0.1139 - auc_14: 0.9107\n",
      "Epoch 6/10\n",
      "794/794 [==============================] - 3s 4ms/step - loss: 0.3562 - accuracy: 0.8424 - mae: 0.2254 - mse: 0.1122 - auc_14: 0.9136\n",
      "Epoch 7/10\n",
      "794/794 [==============================] - 3s 4ms/step - loss: 0.3563 - accuracy: 0.8413 - mae: 0.2256 - mse: 0.1125 - auc_14: 0.9135\n",
      "Epoch 8/10\n",
      "794/794 [==============================] - 3s 4ms/step - loss: 0.3538 - accuracy: 0.8442 - mae: 0.2234 - mse: 0.1113 - auc_14: 0.9148\n",
      "Epoch 9/10\n",
      "794/794 [==============================] - 3s 4ms/step - loss: 0.3509 - accuracy: 0.8463 - mae: 0.2212 - mse: 0.1101 - auc_14: 0.9163\n",
      "Epoch 10/10\n",
      "794/794 [==============================] - 3s 4ms/step - loss: 0.3503 - accuracy: 0.8454 - mae: 0.2208 - mse: 0.1101 - auc_14: 0.9167\n",
      "84\n",
      "84\n",
      "84\n",
      "84\n",
      "84\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "Epoch 1/10\n",
      "794/794 [==============================] - 3s 4ms/step - loss: 0.3557 - accuracy: 0.8444 - mae: 0.2243 - mse: 0.1119 - auc_14: 0.9138\n",
      "Epoch 2/10\n",
      "794/794 [==============================] - 3s 4ms/step - loss: 0.3591 - accuracy: 0.8422 - mae: 0.2269 - mse: 0.1133 - auc_14: 0.9119\n",
      "Epoch 3/10\n",
      "794/794 [==============================] - 3s 4ms/step - loss: 0.3580 - accuracy: 0.8434 - mae: 0.2264 - mse: 0.1129 - auc_14: 0.9126\n",
      "Epoch 4/10\n",
      "794/794 [==============================] - 3s 4ms/step - loss: 0.3567 - accuracy: 0.8438 - mae: 0.2252 - mse: 0.1124 - auc_14: 0.9133\n",
      "Epoch 5/10\n",
      "794/794 [==============================] - 3s 4ms/step - loss: 0.3516 - accuracy: 0.8457 - mae: 0.2219 - mse: 0.1108 - auc_14: 0.9159\n",
      "Epoch 6/10\n",
      "794/794 [==============================] - 3s 4ms/step - loss: 0.3508 - accuracy: 0.8455 - mae: 0.2217 - mse: 0.1106 - auc_14: 0.9163\n",
      "Epoch 7/10\n",
      "794/794 [==============================] - 3s 4ms/step - loss: 0.3527 - accuracy: 0.8458 - mae: 0.2229 - mse: 0.1113 - auc_14: 0.9153\n",
      "Epoch 8/10\n",
      "794/794 [==============================] - 3s 4ms/step - loss: 0.3507 - accuracy: 0.8455 - mae: 0.2218 - mse: 0.1108 - auc_14: 0.9162\n",
      "Epoch 9/10\n",
      "794/794 [==============================] - 3s 4ms/step - loss: 0.3528 - accuracy: 0.8448 - mae: 0.2231 - mse: 0.1113 - auc_14: 0.9153\n",
      "Epoch 10/10\n",
      "794/794 [==============================] - 3s 4ms/step - loss: 0.3599 - accuracy: 0.8403 - mae: 0.2287 - mse: 0.1141 - auc_14: 0.9116\n",
      "84\n",
      "84\n",
      "84\n",
      "84\n",
      "84\n",
      "84\n",
      "84\n",
      "84\n",
      "84\n",
      "3\n",
      "244/244 [==============================] - 0s 636us/step - loss: 0.3090 - accuracy: 0.8622 - mae: 0.2056 - mse: 0.0965 - auc_14: 0.8729\n",
      "train positive label: 5272 - train negative label: 64879\n",
      "up and down sampling => train positive label: 36904 - train negative label: 64879\n",
      "Test positive label: 590 - Test negative label: 7205\n",
      "Epoch 1/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.5101 - accuracy: 0.7578 - mae: 0.3362 - mse: 0.1674 - auc_15: 0.8092\n",
      "Epoch 2/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4654 - accuracy: 0.7851 - mae: 0.3023 - mse: 0.1501 - auc_15: 0.8442\n",
      "Epoch 3/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4448 - accuracy: 0.7961 - mae: 0.2872 - mse: 0.1426 - auc_15: 0.8591\n",
      "Epoch 4/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4333 - accuracy: 0.8040 - mae: 0.2786 - mse: 0.1384 - auc_15: 0.8673\n",
      "Epoch 5/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4227 - accuracy: 0.8082 - mae: 0.2708 - mse: 0.1347 - auc_15: 0.8744\n",
      "Epoch 6/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4130 - accuracy: 0.8142 - mae: 0.2637 - mse: 0.1312 - auc_15: 0.8808\n",
      "Epoch 7/10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4073 - accuracy: 0.8181 - mae: 0.2597 - mse: 0.1291 - auc_15: 0.8841\n",
      "Epoch 8/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3980 - accuracy: 0.8227 - mae: 0.2532 - mse: 0.1261 - auc_15: 0.8898\n",
      "Epoch 9/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3945 - accuracy: 0.8251 - mae: 0.2502 - mse: 0.1246 - auc_15: 0.8920\n",
      "Epoch 10/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3879 - accuracy: 0.8288 - mae: 0.2460 - mse: 0.1225 - auc_15: 0.8959\n",
      "82\n",
      "82\n",
      "81\n",
      "81\n",
      "80\n",
      "80\n",
      "79\n",
      "78\n",
      "75\n",
      "Epoch 1/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3831 - accuracy: 0.8311 - mae: 0.2426 - mse: 0.1207 - auc_15: 0.8986\n",
      "Epoch 2/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3778 - accuracy: 0.8330 - mae: 0.2394 - mse: 0.1193 - auc_15: 0.9018\n",
      "Epoch 3/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3750 - accuracy: 0.8346 - mae: 0.2372 - mse: 0.1182 - auc_15: 0.9032\n",
      "Epoch 4/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3725 - accuracy: 0.8357 - mae: 0.2356 - mse: 0.1174 - auc_15: 0.9048\n",
      "Epoch 5/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3696 - accuracy: 0.8366 - mae: 0.2339 - mse: 0.1165 - auc_15: 0.9064\n",
      "Epoch 6/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3644 - accuracy: 0.8391 - mae: 0.2303 - mse: 0.1149 - auc_15: 0.9093\n",
      "Epoch 7/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3612 - accuracy: 0.8400 - mae: 0.2287 - mse: 0.1139 - auc_15: 0.9111\n",
      "Epoch 8/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3646 - accuracy: 0.8394 - mae: 0.2299 - mse: 0.1148 - auc_15: 0.9093\n",
      "Epoch 9/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3597 - accuracy: 0.8423 - mae: 0.2270 - mse: 0.1130 - auc_15: 0.9120\n",
      "Epoch 10/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3605 - accuracy: 0.8412 - mae: 0.2277 - mse: 0.1136 - auc_15: 0.9116\n",
      "84\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "Epoch 1/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3626 - accuracy: 0.8403 - mae: 0.2289 - mse: 0.1144 - auc_15: 0.9105\n",
      "Epoch 2/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3607 - accuracy: 0.8389 - mae: 0.2288 - mse: 0.1140 - auc_15: 0.9115\n",
      "Epoch 3/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3602 - accuracy: 0.8393 - mae: 0.2281 - mse: 0.1140 - auc_15: 0.9117\n",
      "Epoch 4/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3636 - accuracy: 0.8382 - mae: 0.2305 - mse: 0.1149 - auc_15: 0.9100\n",
      "Epoch 5/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3593 - accuracy: 0.8416 - mae: 0.2270 - mse: 0.1133 - auc_15: 0.9123\n",
      "Epoch 6/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3584 - accuracy: 0.8419 - mae: 0.2269 - mse: 0.1130 - auc_15: 0.9128\n",
      "Epoch 7/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3630 - accuracy: 0.8380 - mae: 0.2302 - mse: 0.1150 - auc_15: 0.9102\n",
      "Epoch 8/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3623 - accuracy: 0.8385 - mae: 0.2299 - mse: 0.1148 - auc_15: 0.9107\n",
      "Epoch 9/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3695 - accuracy: 0.8351 - mae: 0.2339 - mse: 0.1169 - auc_15: 0.9068\n",
      "Epoch 10/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3699 - accuracy: 0.8354 - mae: 0.2348 - mse: 0.1171 - auc_15: 0.9065\n",
      "83\n",
      "83\n",
      "83\n",
      "84\n",
      "84\n",
      "83\n",
      "83\n",
      "83\n",
      "84\n",
      "Epoch 1/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3691 - accuracy: 0.8354 - mae: 0.2343 - mse: 0.1170 - auc_15: 0.9069\n",
      "Epoch 2/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3685 - accuracy: 0.8363 - mae: 0.2338 - mse: 0.1167 - auc_15: 0.9073\n",
      "Epoch 3/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3692 - accuracy: 0.8340 - mae: 0.2345 - mse: 0.1172 - auc_15: 0.9068\n",
      "Epoch 4/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3673 - accuracy: 0.8355 - mae: 0.2334 - mse: 0.1166 - auc_15: 0.9078\n",
      "Epoch 5/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3659 - accuracy: 0.8366 - mae: 0.2321 - mse: 0.1159 - auc_15: 0.9086\n",
      "Epoch 6/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3646 - accuracy: 0.8374 - mae: 0.2315 - mse: 0.1156 - auc_15: 0.9094\n",
      "Epoch 7/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3615 - accuracy: 0.8391 - mae: 0.2292 - mse: 0.1145 - auc_15: 0.9109\n",
      "Epoch 8/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3599 - accuracy: 0.8382 - mae: 0.2288 - mse: 0.1142 - auc_15: 0.9117\n",
      "Epoch 9/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3603 - accuracy: 0.8390 - mae: 0.2286 - mse: 0.1142 - auc_15: 0.9117\n",
      "Epoch 10/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3583 - accuracy: 0.8387 - mae: 0.2276 - mse: 0.1135 - auc_15: 0.9127\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "4\n",
      "244/244 [==============================] - 0s 648us/step - loss: 0.2773 - accuracy: 0.8745 - mae: 0.1803 - mse: 0.0859 - auc_15: 0.8670\n",
      "train positive label: 5302 - train negative label: 64850\n",
      "up and down sampling => train positive label: 37114 - train negative label: 64850\n",
      "Test positive label: 560 - Test negative label: 7234\n",
      "Epoch 1/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.5077 - accuracy: 0.7589 - mae: 0.3341 - mse: 0.1666 - auc_16: 0.8102\n",
      "Epoch 2/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.4650 - accuracy: 0.7855 - mae: 0.3017 - mse: 0.1501 - auc_16: 0.8445\n",
      "Epoch 3/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.4498 - accuracy: 0.7954 - mae: 0.2900 - mse: 0.1443 - auc_16: 0.8558\n",
      "Epoch 4/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.4384 - accuracy: 0.8017 - mae: 0.2820 - mse: 0.1402 - auc_16: 0.8637\n",
      "Epoch 5/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.4292 - accuracy: 0.8075 - mae: 0.2748 - mse: 0.1369 - auc_16: 0.8701\n",
      "Epoch 6/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.4197 - accuracy: 0.8121 - mae: 0.2683 - mse: 0.1334 - auc_16: 0.8766\n",
      "Epoch 7/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.4104 - accuracy: 0.8168 - mae: 0.2619 - mse: 0.1304 - auc_16: 0.8824\n",
      "Epoch 8/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.4049 - accuracy: 0.8199 - mae: 0.2577 - mse: 0.1282 - auc_16: 0.8860\n",
      "Epoch 9/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3977 - accuracy: 0.8249 - mae: 0.2524 - mse: 0.1257 - auc_16: 0.8902\n",
      "Epoch 10/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3931 - accuracy: 0.8269 - mae: 0.2494 - mse: 0.1243 - auc_16: 0.8931\n",
      "82\n",
      "81\n",
      "81\n",
      "81\n",
      "80\n",
      "80\n",
      "79\n",
      "78\n",
      "75\n",
      "Epoch 1/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3864 - accuracy: 0.8300 - mae: 0.2454 - mse: 0.1220 - auc_16: 0.8969\n",
      "Epoch 2/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3821 - accuracy: 0.8316 - mae: 0.2421 - mse: 0.1208 - auc_16: 0.8994\n",
      "Epoch 3/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3747 - accuracy: 0.8358 - mae: 0.2369 - mse: 0.1179 - auc_16: 0.9039\n",
      "Epoch 4/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3726 - accuracy: 0.8354 - mae: 0.2356 - mse: 0.1175 - auc_16: 0.9050\n",
      "Epoch 5/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3671 - accuracy: 0.8373 - mae: 0.2324 - mse: 0.1157 - auc_16: 0.9081\n",
      "Epoch 6/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3645 - accuracy: 0.8379 - mae: 0.2309 - mse: 0.1151 - auc_16: 0.9095\n",
      "Epoch 7/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3603 - accuracy: 0.8427 - mae: 0.2278 - mse: 0.1135 - auc_16: 0.9115\n",
      "Epoch 8/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3574 - accuracy: 0.8423 - mae: 0.2259 - mse: 0.1126 - auc_16: 0.9133\n",
      "Epoch 9/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3571 - accuracy: 0.8430 - mae: 0.2253 - mse: 0.1123 - auc_16: 0.9135\n",
      "Epoch 10/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3515 - accuracy: 0.8460 - mae: 0.2219 - mse: 0.1105 - auc_16: 0.9163\n",
      "84\n",
      "84\n",
      "84\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "Epoch 1/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3528 - accuracy: 0.8446 - mae: 0.2227 - mse: 0.1111 - auc_16: 0.9157\n",
      "Epoch 2/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3514 - accuracy: 0.8452 - mae: 0.2220 - mse: 0.1107 - auc_16: 0.9165\n",
      "Epoch 3/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3506 - accuracy: 0.8458 - mae: 0.2215 - mse: 0.1105 - auc_16: 0.9168\n",
      "Epoch 4/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3534 - accuracy: 0.8433 - mae: 0.2237 - mse: 0.1116 - auc_16: 0.9156\n",
      "Epoch 5/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3559 - accuracy: 0.8411 - mae: 0.2262 - mse: 0.1127 - auc_16: 0.9142\n",
      "Epoch 6/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3551 - accuracy: 0.8417 - mae: 0.2252 - mse: 0.1123 - auc_16: 0.9147\n",
      "Epoch 7/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3528 - accuracy: 0.8434 - mae: 0.2233 - mse: 0.1114 - auc_16: 0.9158\n",
      "Epoch 8/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3541 - accuracy: 0.8436 - mae: 0.2242 - mse: 0.1119 - auc_16: 0.9151\n",
      "Epoch 9/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3541 - accuracy: 0.8424 - mae: 0.2247 - mse: 0.1121 - auc_16: 0.9151\n",
      "Epoch 10/10\n",
      "797/797 [==============================] - 3s 4ms/step - loss: 0.3562 - accuracy: 0.8412 - mae: 0.2261 - mse: 0.1128 - auc_16: 0.9140\n",
      "84\n",
      "84\n",
      "84\n",
      "84\n",
      "84\n",
      "84\n",
      "84\n",
      "84\n",
      "84\n",
      "3\n",
      "244/244 [==============================] - 0s 632us/step - loss: 0.2692 - accuracy: 0.8867 - mae: 0.1827 - mse: 0.0815 - auc_16: 0.8893\n",
      "train positive label: 5258 - train negative label: 64894\n",
      "up and down sampling => train positive label: 36806 - train negative label: 64894\n",
      "Test positive label: 604 - Test negative label: 7190\n",
      "Epoch 1/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.5062 - accuracy: 0.7604 - mae: 0.3350 - mse: 0.1660 - auc_17: 0.8128\n",
      "Epoch 2/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.4581 - accuracy: 0.7886 - mae: 0.2975 - mse: 0.1476 - auc_17: 0.8501\n",
      "Epoch 3/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.4397 - accuracy: 0.8004 - mae: 0.2833 - mse: 0.1407 - auc_17: 0.8630\n",
      "Epoch 4/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.4277 - accuracy: 0.8066 - mae: 0.2746 - mse: 0.1364 - auc_17: 0.8709\n",
      "Epoch 5/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.4176 - accuracy: 0.8122 - mae: 0.2669 - mse: 0.1329 - auc_17: 0.8776\n",
      "Epoch 6/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.4104 - accuracy: 0.8181 - mae: 0.2619 - mse: 0.1302 - auc_17: 0.8820\n",
      "Epoch 7/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.4028 - accuracy: 0.8209 - mae: 0.2564 - mse: 0.1276 - auc_17: 0.8871\n",
      "Epoch 8/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3958 - accuracy: 0.8252 - mae: 0.2513 - mse: 0.1252 - auc_17: 0.8912\n",
      "Epoch 9/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3896 - accuracy: 0.8277 - mae: 0.2469 - mse: 0.1231 - auc_17: 0.8950\n",
      "Epoch 10/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3857 - accuracy: 0.8304 - mae: 0.2444 - mse: 0.1216 - auc_17: 0.8973\n",
      "82\n",
      "82\n",
      "82\n",
      "81\n",
      "81\n",
      "80\n",
      "80\n",
      "78\n",
      "76\n",
      "Epoch 1/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3804 - accuracy: 0.8324 - mae: 0.2406 - mse: 0.1198 - auc_17: 0.9006\n",
      "Epoch 2/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3752 - accuracy: 0.8354 - mae: 0.2371 - mse: 0.1180 - auc_17: 0.9032\n",
      "Epoch 3/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3696 - accuracy: 0.8386 - mae: 0.2331 - mse: 0.1161 - auc_17: 0.9065\n",
      "Epoch 4/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3667 - accuracy: 0.8397 - mae: 0.2311 - mse: 0.1153 - auc_17: 0.9079\n",
      "Epoch 5/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3637 - accuracy: 0.8408 - mae: 0.2295 - mse: 0.1144 - auc_17: 0.9096\n",
      "Epoch 6/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3634 - accuracy: 0.8399 - mae: 0.2296 - mse: 0.1145 - auc_17: 0.9100\n",
      "Epoch 7/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3602 - accuracy: 0.8424 - mae: 0.2275 - mse: 0.1133 - auc_17: 0.9115\n",
      "Epoch 8/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3560 - accuracy: 0.8438 - mae: 0.2245 - mse: 0.1119 - auc_17: 0.9137\n",
      "Epoch 9/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3513 - accuracy: 0.8454 - mae: 0.2214 - mse: 0.1106 - auc_17: 0.9161\n",
      "Epoch 10/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3499 - accuracy: 0.8471 - mae: 0.2209 - mse: 0.1100 - auc_17: 0.9168\n",
      "84\n",
      "84\n",
      "84\n",
      "83\n",
      "84\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "Epoch 1/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3511 - accuracy: 0.8454 - mae: 0.2218 - mse: 0.1108 - auc_17: 0.9163\n",
      "Epoch 2/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3517 - accuracy: 0.8457 - mae: 0.2224 - mse: 0.1108 - auc_17: 0.9159\n",
      "Epoch 3/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3570 - accuracy: 0.8420 - mae: 0.2261 - mse: 0.1129 - auc_17: 0.9131\n",
      "Epoch 4/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3539 - accuracy: 0.8434 - mae: 0.2239 - mse: 0.1117 - auc_17: 0.9148\n",
      "Epoch 5/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3527 - accuracy: 0.8436 - mae: 0.2229 - mse: 0.1113 - auc_17: 0.9155\n",
      "Epoch 6/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3511 - accuracy: 0.8447 - mae: 0.2220 - mse: 0.1108 - auc_17: 0.9163\n",
      "Epoch 7/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3536 - accuracy: 0.8430 - mae: 0.2241 - mse: 0.1118 - auc_17: 0.9149\n",
      "Epoch 8/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3540 - accuracy: 0.8422 - mae: 0.2244 - mse: 0.1121 - auc_17: 0.9148\n",
      "Epoch 9/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3610 - accuracy: 0.8373 - mae: 0.2300 - mse: 0.1148 - auc_17: 0.9111\n",
      "Epoch 10/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3677 - accuracy: 0.8334 - mae: 0.2349 - mse: 0.1172 - auc_17: 0.9076\n",
      "83\n",
      "84\n",
      "84\n",
      "84\n",
      "84\n",
      "84\n",
      "84\n",
      "84\n",
      "84\n",
      "Epoch 1/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3640 - accuracy: 0.8350 - mae: 0.2323 - mse: 0.1161 - auc_17: 0.9095\n",
      "Epoch 2/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3665 - accuracy: 0.8324 - mae: 0.2346 - mse: 0.1171 - auc_17: 0.9082\n",
      "Epoch 3/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3628 - accuracy: 0.8346 - mae: 0.2321 - mse: 0.1157 - auc_17: 0.9102\n",
      "Epoch 4/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3624 - accuracy: 0.8360 - mae: 0.2311 - mse: 0.1156 - auc_17: 0.9103\n",
      "Epoch 5/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3624 - accuracy: 0.8353 - mae: 0.2316 - mse: 0.1156 - auc_17: 0.9103\n",
      "Epoch 6/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3562 - accuracy: 0.8379 - mae: 0.2278 - mse: 0.1138 - auc_17: 0.9134\n",
      "Epoch 7/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3541 - accuracy: 0.8385 - mae: 0.2264 - mse: 0.1128 - auc_17: 0.9146\n",
      "Epoch 8/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3555 - accuracy: 0.8384 - mae: 0.2270 - mse: 0.1134 - auc_17: 0.9139\n",
      "Epoch 9/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3549 - accuracy: 0.8388 - mae: 0.2269 - mse: 0.1132 - auc_17: 0.9142\n",
      "Epoch 10/10\n",
      "795/795 [==============================] - 3s 4ms/step - loss: 0.3592 - accuracy: 0.8366 - mae: 0.2299 - mse: 0.1149 - auc_17: 0.9118\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "4\n",
      "244/244 [==============================] - 0s 640us/step - loss: 0.2871 - accuracy: 0.8628 - mae: 0.1912 - mse: 0.0889 - auc_17: 0.8677\n",
      "train positive label: 5270 - train negative label: 64882\n",
      "up and down sampling => train positive label: 36890 - train negative label: 64882\n",
      "Test positive label: 592 - Test negative label: 7202\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.5079 - accuracy: 0.7584 - mae: 0.3347 - mse: 0.1661 - auc_18: 0.8123\n",
      "Epoch 2/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4638 - accuracy: 0.7866 - mae: 0.3010 - mse: 0.1494 - auc_18: 0.8459\n",
      "Epoch 3/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4468 - accuracy: 0.7954 - mae: 0.2887 - mse: 0.1433 - auc_18: 0.8582\n",
      "Epoch 4/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4337 - accuracy: 0.8030 - mae: 0.2789 - mse: 0.1386 - auc_18: 0.8668\n",
      "Epoch 5/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4236 - accuracy: 0.8093 - mae: 0.2709 - mse: 0.1349 - auc_18: 0.8734\n",
      "Epoch 6/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4156 - accuracy: 0.8140 - mae: 0.2657 - mse: 0.1321 - auc_18: 0.8786\n",
      "Epoch 7/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4059 - accuracy: 0.8189 - mae: 0.2583 - mse: 0.1285 - auc_18: 0.8849\n",
      "Epoch 8/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4005 - accuracy: 0.8213 - mae: 0.2542 - mse: 0.1266 - auc_18: 0.8884\n",
      "Epoch 9/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3951 - accuracy: 0.8259 - mae: 0.2504 - mse: 0.1246 - auc_18: 0.8915\n",
      "Epoch 10/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3896 - accuracy: 0.8288 - mae: 0.2464 - mse: 0.1226 - auc_18: 0.8950\n",
      "82\n",
      "82\n",
      "81\n",
      "81\n",
      "80\n",
      "80\n",
      "79\n",
      "78\n",
      "75\n",
      "Epoch 1/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3871 - accuracy: 0.8293 - mae: 0.2452 - mse: 0.1220 - auc_18: 0.8965\n",
      "Epoch 2/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3813 - accuracy: 0.8320 - mae: 0.2409 - mse: 0.1201 - auc_18: 0.8996\n",
      "Epoch 3/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3781 - accuracy: 0.8332 - mae: 0.2386 - mse: 0.1190 - auc_18: 0.9016\n",
      "Epoch 4/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3786 - accuracy: 0.8333 - mae: 0.2392 - mse: 0.1193 - auc_18: 0.9012\n",
      "Epoch 5/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3770 - accuracy: 0.8323 - mae: 0.2380 - mse: 0.1186 - auc_18: 0.9022\n",
      "Epoch 6/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3738 - accuracy: 0.8344 - mae: 0.2362 - mse: 0.1177 - auc_18: 0.9041\n",
      "Epoch 7/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3713 - accuracy: 0.8361 - mae: 0.2343 - mse: 0.1170 - auc_18: 0.9055\n",
      "Epoch 8/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3703 - accuracy: 0.8366 - mae: 0.2337 - mse: 0.1166 - auc_18: 0.9061\n",
      "Epoch 9/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3653 - accuracy: 0.8399 - mae: 0.2298 - mse: 0.1148 - auc_18: 0.9086\n",
      "Epoch 10/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3615 - accuracy: 0.8400 - mae: 0.2276 - mse: 0.1137 - auc_18: 0.9108\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "82\n",
      "Epoch 1/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3597 - accuracy: 0.8419 - mae: 0.2263 - mse: 0.1129 - auc_18: 0.9118\n",
      "Epoch 2/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3602 - accuracy: 0.8417 - mae: 0.2268 - mse: 0.1131 - auc_18: 0.9115\n",
      "Epoch 3/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3581 - accuracy: 0.8420 - mae: 0.2253 - mse: 0.1124 - auc_18: 0.9127\n",
      "Epoch 4/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3545 - accuracy: 0.8438 - mae: 0.2229 - mse: 0.1113 - auc_18: 0.9145\n",
      "Epoch 5/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3544 - accuracy: 0.8451 - mae: 0.2228 - mse: 0.1112 - auc_18: 0.9145\n",
      "Epoch 6/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3533 - accuracy: 0.8441 - mae: 0.2220 - mse: 0.1110 - auc_18: 0.9151\n",
      "Epoch 7/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3512 - accuracy: 0.8451 - mae: 0.2212 - mse: 0.1104 - auc_18: 0.9163\n",
      "Epoch 8/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3536 - accuracy: 0.8435 - mae: 0.2228 - mse: 0.1113 - auc_18: 0.9149\n",
      "Epoch 9/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3632 - accuracy: 0.8398 - mae: 0.2285 - mse: 0.1143 - auc_18: 0.9100\n",
      "Epoch 10/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3705 - accuracy: 0.8359 - mae: 0.2338 - mse: 0.1169 - auc_18: 0.9060\n",
      "83\n",
      "84\n",
      "84\n",
      "84\n",
      "84\n",
      "84\n",
      "84\n",
      "84\n",
      "84\n",
      "Epoch 1/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3685 - accuracy: 0.8356 - mae: 0.2332 - mse: 0.1164 - auc_18: 0.9072\n",
      "Epoch 2/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3682 - accuracy: 0.8371 - mae: 0.2329 - mse: 0.1163 - auc_18: 0.9074\n",
      "Epoch 3/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3684 - accuracy: 0.8364 - mae: 0.2332 - mse: 0.1164 - auc_18: 0.9073\n",
      "Epoch 4/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3815 - accuracy: 0.8296 - mae: 0.2421 - mse: 0.1210 - auc_18: 0.9002\n",
      "Epoch 5/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3790 - accuracy: 0.8284 - mae: 0.2414 - mse: 0.1204 - auc_18: 0.9016\n",
      "Epoch 6/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3776 - accuracy: 0.8296 - mae: 0.2404 - mse: 0.1201 - auc_18: 0.9021\n",
      "Epoch 7/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3761 - accuracy: 0.8299 - mae: 0.2393 - mse: 0.1194 - auc_18: 0.9032\n",
      "Epoch 8/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3869 - accuracy: 0.8237 - mae: 0.2470 - mse: 0.1234 - auc_18: 0.8971\n",
      "Epoch 9/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3901 - accuracy: 0.8218 - mae: 0.2499 - mse: 0.1248 - auc_18: 0.8952\n",
      "Epoch 10/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3970 - accuracy: 0.8172 - mae: 0.2550 - mse: 0.1274 - auc_18: 0.8911\n",
      "82\n",
      "82\n",
      "82\n",
      "82\n",
      "82\n",
      "82\n",
      "83\n",
      "83\n",
      "83\n",
      "Epoch 1/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3971 - accuracy: 0.8172 - mae: 0.2546 - mse: 0.1273 - auc_18: 0.8910\n",
      "Epoch 2/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3982 - accuracy: 0.8159 - mae: 0.2560 - mse: 0.1278 - auc_18: 0.8903\n",
      "Epoch 3/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4112 - accuracy: 0.8083 - mae: 0.2660 - mse: 0.1328 - auc_18: 0.8823\n",
      "Epoch 4/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4106 - accuracy: 0.8087 - mae: 0.2657 - mse: 0.1328 - auc_18: 0.8826\n",
      "Epoch 5/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4088 - accuracy: 0.8087 - mae: 0.2645 - mse: 0.1322 - auc_18: 0.8834\n",
      "Epoch 6/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4075 - accuracy: 0.8093 - mae: 0.2640 - mse: 0.1318 - auc_18: 0.8843\n",
      "Epoch 7/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4062 - accuracy: 0.8090 - mae: 0.2633 - mse: 0.1314 - auc_18: 0.8852\n",
      "Epoch 8/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4280 - accuracy: 0.7973 - mae: 0.2782 - mse: 0.1390 - auc_18: 0.8714\n",
      "Epoch 9/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4274 - accuracy: 0.7961 - mae: 0.2781 - mse: 0.1391 - auc_18: 0.8716\n",
      "Epoch 10/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4264 - accuracy: 0.7986 - mae: 0.2776 - mse: 0.1387 - auc_18: 0.8722\n",
      "79\n",
      "79\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "81\n",
      "81\n",
      "Epoch 1/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4256 - accuracy: 0.8004 - mae: 0.2765 - mse: 0.1382 - auc_18: 0.8728\n",
      "Epoch 2/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4232 - accuracy: 0.7989 - mae: 0.2755 - mse: 0.1376 - auc_18: 0.8742\n",
      "Epoch 3/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4245 - accuracy: 0.7992 - mae: 0.2761 - mse: 0.1381 - auc_18: 0.8734\n",
      "Epoch 4/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4204 - accuracy: 0.7999 - mae: 0.2743 - mse: 0.1371 - auc_18: 0.8756\n",
      "Epoch 5/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4207 - accuracy: 0.7997 - mae: 0.2741 - mse: 0.1371 - auc_18: 0.8757\n",
      "Epoch 6/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4200 - accuracy: 0.8008 - mae: 0.2737 - mse: 0.1367 - auc_18: 0.8761\n",
      "Epoch 7/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4186 - accuracy: 0.8011 - mae: 0.2727 - mse: 0.1363 - auc_18: 0.8769\n",
      "Epoch 8/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4177 - accuracy: 0.8016 - mae: 0.2720 - mse: 0.1359 - auc_18: 0.8775\n",
      "Epoch 9/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4174 - accuracy: 0.8016 - mae: 0.2722 - mse: 0.1359 - auc_18: 0.8776\n",
      "Epoch 10/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4154 - accuracy: 0.8035 - mae: 0.2706 - mse: 0.1352 - auc_18: 0.8788\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "80\n",
      "Epoch 1/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4136 - accuracy: 0.8037 - mae: 0.2694 - mse: 0.1346 - auc_18: 0.8799\n",
      "Epoch 2/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4139 - accuracy: 0.8027 - mae: 0.2703 - mse: 0.1350 - auc_18: 0.8796\n",
      "Epoch 3/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4140 - accuracy: 0.8036 - mae: 0.2696 - mse: 0.1348 - auc_18: 0.8797\n",
      "Epoch 4/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4122 - accuracy: 0.8045 - mae: 0.2685 - mse: 0.1342 - auc_18: 0.8807\n",
      "Epoch 5/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4118 - accuracy: 0.8043 - mae: 0.2687 - mse: 0.1342 - auc_18: 0.8810\n",
      "Epoch 6/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4110 - accuracy: 0.8044 - mae: 0.2685 - mse: 0.1341 - auc_18: 0.8812\n",
      "Epoch 7/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4111 - accuracy: 0.8042 - mae: 0.2679 - mse: 0.1341 - auc_18: 0.8812\n",
      "Epoch 8/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4093 - accuracy: 0.8058 - mae: 0.2673 - mse: 0.1334 - auc_18: 0.8823\n",
      "Epoch 9/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4088 - accuracy: 0.8052 - mae: 0.2668 - mse: 0.1334 - auc_18: 0.8825\n",
      "Epoch 10/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4096 - accuracy: 0.8054 - mae: 0.2675 - mse: 0.1335 - auc_18: 0.8820\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "7\n",
      "244/244 [==============================] - 0s 640us/step - loss: 0.3440 - accuracy: 0.8418 - mae: 0.2264 - mse: 0.1084 - auc_18: 0.8096\n",
      "train positive label: 5272 - train negative label: 64880\n",
      "up and down sampling => train positive label: 36904 - train negative label: 64880\n",
      "Test positive label: 590 - Test negative label: 7204\n",
      "Epoch 1/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.5096 - accuracy: 0.7586 - mae: 0.3367 - mse: 0.1673 - auc_19: 0.8096\n",
      "Epoch 2/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4662 - accuracy: 0.7884 - mae: 0.3026 - mse: 0.1502 - auc_19: 0.8433\n",
      "Epoch 3/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4485 - accuracy: 0.7958 - mae: 0.2898 - mse: 0.1440 - auc_19: 0.8560\n",
      "Epoch 4/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4369 - accuracy: 0.8041 - mae: 0.2809 - mse: 0.1396 - auc_19: 0.8643\n",
      "Epoch 5/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4263 - accuracy: 0.8097 - mae: 0.2731 - mse: 0.1358 - auc_19: 0.8715\n",
      "Epoch 6/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4182 - accuracy: 0.8144 - mae: 0.2671 - mse: 0.1327 - auc_19: 0.8771\n",
      "Epoch 7/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4092 - accuracy: 0.8181 - mae: 0.2607 - mse: 0.1298 - auc_19: 0.8828\n",
      "Epoch 8/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4010 - accuracy: 0.8232 - mae: 0.2549 - mse: 0.1268 - auc_19: 0.8878\n",
      "Epoch 9/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3946 - accuracy: 0.8259 - mae: 0.2502 - mse: 0.1246 - auc_19: 0.8919\n",
      "Epoch 10/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3886 - accuracy: 0.8295 - mae: 0.2460 - mse: 0.1223 - auc_19: 0.8957\n",
      "82\n",
      "82\n",
      "81\n",
      "81\n",
      "80\n",
      "80\n",
      "79\n",
      "78\n",
      "75\n",
      "Epoch 1/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3835 - accuracy: 0.8299 - mae: 0.2430 - mse: 0.1209 - auc_19: 0.8985\n",
      "Epoch 2/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3780 - accuracy: 0.8342 - mae: 0.2387 - mse: 0.1188 - auc_19: 0.9016\n",
      "Epoch 3/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3723 - accuracy: 0.8365 - mae: 0.2353 - mse: 0.1171 - auc_19: 0.9048\n",
      "Epoch 4/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3705 - accuracy: 0.8366 - mae: 0.2340 - mse: 0.1167 - auc_19: 0.9062\n",
      "Epoch 5/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3655 - accuracy: 0.8391 - mae: 0.2311 - mse: 0.1152 - auc_19: 0.9086\n",
      "Epoch 6/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3616 - accuracy: 0.8406 - mae: 0.2282 - mse: 0.1139 - auc_19: 0.9108\n",
      "Epoch 7/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3597 - accuracy: 0.8429 - mae: 0.2269 - mse: 0.1130 - auc_19: 0.9120\n",
      "Epoch 8/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3575 - accuracy: 0.8426 - mae: 0.2256 - mse: 0.1123 - auc_19: 0.9133\n",
      "Epoch 9/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3578 - accuracy: 0.8423 - mae: 0.2259 - mse: 0.1126 - auc_19: 0.9130\n",
      "Epoch 10/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3543 - accuracy: 0.8431 - mae: 0.2236 - mse: 0.1116 - auc_19: 0.9148\n",
      "84\n",
      "84\n",
      "84\n",
      "84\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "82\n",
      "Epoch 1/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3627 - accuracy: 0.8387 - mae: 0.2302 - mse: 0.1146 - auc_19: 0.9106\n",
      "Epoch 2/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3608 - accuracy: 0.8404 - mae: 0.2284 - mse: 0.1141 - auc_19: 0.9116\n",
      "Epoch 3/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3584 - accuracy: 0.8398 - mae: 0.2273 - mse: 0.1135 - auc_19: 0.9128\n",
      "Epoch 4/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3606 - accuracy: 0.8401 - mae: 0.2287 - mse: 0.1142 - auc_19: 0.9116\n",
      "Epoch 5/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3578 - accuracy: 0.8414 - mae: 0.2270 - mse: 0.1132 - auc_19: 0.9130\n",
      "Epoch 6/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3566 - accuracy: 0.8417 - mae: 0.2264 - mse: 0.1129 - auc_19: 0.9137\n",
      "Epoch 7/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3544 - accuracy: 0.8428 - mae: 0.2248 - mse: 0.1120 - auc_19: 0.9148\n",
      "Epoch 8/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3622 - accuracy: 0.8373 - mae: 0.2307 - mse: 0.1151 - auc_19: 0.9106\n",
      "Epoch 9/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3603 - accuracy: 0.8386 - mae: 0.2293 - mse: 0.1145 - auc_19: 0.9115\n",
      "Epoch 10/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3588 - accuracy: 0.8390 - mae: 0.2279 - mse: 0.1138 - auc_19: 0.9124\n",
      "83\n",
      "83\n",
      "84\n",
      "84\n",
      "84\n",
      "84\n",
      "83\n",
      "84\n",
      "83\n",
      "Epoch 1/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3562 - accuracy: 0.8397 - mae: 0.2268 - mse: 0.1129 - auc_19: 0.9137\n",
      "Epoch 2/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3556 - accuracy: 0.8401 - mae: 0.2265 - mse: 0.1132 - auc_19: 0.9139\n",
      "Epoch 3/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3558 - accuracy: 0.8409 - mae: 0.2262 - mse: 0.1129 - auc_19: 0.9140\n",
      "Epoch 4/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3612 - accuracy: 0.8366 - mae: 0.2306 - mse: 0.1152 - auc_19: 0.9110\n",
      "Epoch 5/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3728 - accuracy: 0.8310 - mae: 0.2385 - mse: 0.1191 - auc_19: 0.9048\n",
      "Epoch 6/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3732 - accuracy: 0.8305 - mae: 0.2394 - mse: 0.1193 - auc_19: 0.9046\n",
      "Epoch 7/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3764 - accuracy: 0.8291 - mae: 0.2413 - mse: 0.1205 - auc_19: 0.9028\n",
      "Epoch 8/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3742 - accuracy: 0.8301 - mae: 0.2402 - mse: 0.1199 - auc_19: 0.9038\n",
      "Epoch 9/10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3716 - accuracy: 0.8313 - mae: 0.2380 - mse: 0.1188 - auc_19: 0.9053\n",
      "Epoch 10/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3711 - accuracy: 0.8312 - mae: 0.2380 - mse: 0.1188 - auc_19: 0.9056\n",
      "83\n",
      "83\n",
      "82\n",
      "83\n",
      "83\n",
      "83\n",
      "84\n",
      "84\n",
      "83\n",
      "Epoch 1/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3704 - accuracy: 0.8321 - mae: 0.2375 - mse: 0.1186 - auc_19: 0.9060\n",
      "Epoch 2/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3668 - accuracy: 0.8329 - mae: 0.2350 - mse: 0.1173 - auc_19: 0.9080\n",
      "Epoch 3/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3653 - accuracy: 0.8338 - mae: 0.2344 - mse: 0.1169 - auc_19: 0.9086\n",
      "Epoch 4/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3715 - accuracy: 0.8319 - mae: 0.2380 - mse: 0.1189 - auc_19: 0.9053\n",
      "Epoch 5/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3799 - accuracy: 0.8255 - mae: 0.2446 - mse: 0.1221 - auc_19: 0.9004\n",
      "Epoch 6/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3900 - accuracy: 0.8189 - mae: 0.2526 - mse: 0.1262 - auc_19: 0.8943\n",
      "Epoch 7/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3902 - accuracy: 0.8178 - mae: 0.2530 - mse: 0.1265 - auc_19: 0.8940\n",
      "Epoch 8/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3896 - accuracy: 0.8183 - mae: 0.2526 - mse: 0.1260 - auc_19: 0.8947\n",
      "Epoch 9/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3875 - accuracy: 0.8187 - mae: 0.2513 - mse: 0.1255 - auc_19: 0.8956\n",
      "Epoch 10/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3910 - accuracy: 0.8168 - mae: 0.2538 - mse: 0.1266 - auc_19: 0.8937\n",
      "81\n",
      "81\n",
      "81\n",
      "81\n",
      "82\n",
      "83\n",
      "83\n",
      "83\n",
      "83\n",
      "Epoch 1/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4008 - accuracy: 0.8117 - mae: 0.2604 - mse: 0.1300 - auc_19: 0.8879\n",
      "Epoch 2/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4067 - accuracy: 0.8099 - mae: 0.2645 - mse: 0.1320 - auc_19: 0.8845\n",
      "Epoch 3/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4093 - accuracy: 0.8068 - mae: 0.2665 - mse: 0.1332 - auc_19: 0.8827\n",
      "Epoch 4/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4075 - accuracy: 0.8072 - mae: 0.2656 - mse: 0.1326 - auc_19: 0.8836\n",
      "Epoch 5/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4064 - accuracy: 0.8082 - mae: 0.2649 - mse: 0.1321 - auc_19: 0.8845\n",
      "Epoch 6/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4053 - accuracy: 0.8081 - mae: 0.2639 - mse: 0.1319 - auc_19: 0.8850\n",
      "Epoch 7/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4049 - accuracy: 0.8075 - mae: 0.2640 - mse: 0.1318 - auc_19: 0.8851\n",
      "Epoch 8/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4032 - accuracy: 0.8096 - mae: 0.2628 - mse: 0.1312 - auc_19: 0.8861\n",
      "Epoch 9/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4026 - accuracy: 0.8094 - mae: 0.2620 - mse: 0.1308 - auc_19: 0.8869\n",
      "Epoch 10/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4000 - accuracy: 0.8102 - mae: 0.2604 - mse: 0.1300 - auc_19: 0.8879\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "81\n",
      "Epoch 1/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4012 - accuracy: 0.8095 - mae: 0.2613 - mse: 0.1306 - auc_19: 0.8873\n",
      "Epoch 2/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3994 - accuracy: 0.8108 - mae: 0.2602 - mse: 0.1301 - auc_19: 0.8882\n",
      "Epoch 3/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3980 - accuracy: 0.8113 - mae: 0.2599 - mse: 0.1297 - auc_19: 0.8889\n",
      "Epoch 4/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3979 - accuracy: 0.8113 - mae: 0.2594 - mse: 0.1295 - auc_19: 0.8892\n",
      "Epoch 5/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3980 - accuracy: 0.8114 - mae: 0.2598 - mse: 0.1297 - auc_19: 0.8889\n",
      "Epoch 6/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3971 - accuracy: 0.8107 - mae: 0.2594 - mse: 0.1295 - auc_19: 0.8893\n",
      "Epoch 7/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4007 - accuracy: 0.8089 - mae: 0.2621 - mse: 0.1309 - auc_19: 0.8870\n",
      "Epoch 8/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4087 - accuracy: 0.8038 - mae: 0.2673 - mse: 0.1337 - auc_19: 0.8822\n",
      "Epoch 9/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4063 - accuracy: 0.8053 - mae: 0.2657 - mse: 0.1327 - auc_19: 0.8838\n",
      "Epoch 10/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4065 - accuracy: 0.8062 - mae: 0.2660 - mse: 0.1328 - auc_19: 0.8837\n",
      "80\n",
      "80\n",
      "80\n",
      "81\n",
      "81\n",
      "81\n",
      "81\n",
      "81\n",
      "80\n",
      "Epoch 1/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4049 - accuracy: 0.8047 - mae: 0.2654 - mse: 0.1325 - auc_19: 0.8843\n",
      "Epoch 2/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4046 - accuracy: 0.8054 - mae: 0.2648 - mse: 0.1323 - auc_19: 0.8845\n",
      "Epoch 3/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4034 - accuracy: 0.8065 - mae: 0.2644 - mse: 0.1320 - auc_19: 0.8852\n",
      "Epoch 4/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4041 - accuracy: 0.8048 - mae: 0.2650 - mse: 0.1324 - auc_19: 0.8846\n",
      "Epoch 5/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4027 - accuracy: 0.8072 - mae: 0.2634 - mse: 0.1317 - auc_19: 0.8857\n",
      "Epoch 6/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4016 - accuracy: 0.8075 - mae: 0.2634 - mse: 0.1314 - auc_19: 0.8861\n",
      "Epoch 7/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4011 - accuracy: 0.8082 - mae: 0.2622 - mse: 0.1311 - auc_19: 0.8867\n",
      "Epoch 8/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4007 - accuracy: 0.8081 - mae: 0.2626 - mse: 0.1311 - auc_19: 0.8867\n",
      "Epoch 9/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.3993 - accuracy: 0.8087 - mae: 0.2618 - mse: 0.1307 - auc_19: 0.8873\n",
      "Epoch 10/10\n",
      "796/796 [==============================] - 3s 4ms/step - loss: 0.4016 - accuracy: 0.8084 - mae: 0.2631 - mse: 0.1313 - auc_19: 0.8863\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "8\n",
      "244/244 [==============================] - 0s 636us/step - loss: 0.3202 - accuracy: 0.8471 - mae: 0.2136 - mse: 0.1007 - auc_19: 0.8215\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import KFold\n",
    "\n",
    "Epoch_S = 10\n",
    "\n",
    "def evaluate_model(df, k = 10 , shuffle = False):\n",
    "    result =[]    \n",
    "\n",
    "    kf = KFold(n_splits=10, shuffle= shuffle, random_state=None)\n",
    "    \n",
    "    for train_index, test_index in kf.split(df):\n",
    "\n",
    "        train_ds = [df[index] for index in train_index] \n",
    "        \n",
    "        valid_ds = [df[index] for index in test_index]\n",
    "        \n",
    "        label_pos , label_neg, _, _ = count_lablel(train_ds)\n",
    "        print(f'train positive label: {label_pos} - train negative label: {label_neg}')\n",
    "        \n",
    "        train_ds = up_and_down_Samplenig(train_ds, scale_downsampling = 0.5)\n",
    "        \n",
    "        label_pos , label_neg , _,_ = count_lablel(train_ds)\n",
    "        print(f'up and down sampling => train positive label: {label_pos} - train negative label: {label_neg}')\n",
    "\n",
    "        label_pos , label_neg, _,_ = count_lablel(valid_ds)\n",
    "        print(f'Test positive label: {label_pos} - Test negative label: {label_neg}')\n",
    "\n",
    "        l_train = []\n",
    "        r_train = []\n",
    "        lbls_train = []\n",
    "        l_valid = []\n",
    "        r_valid = []\n",
    "        lbls_valid = []\n",
    "\n",
    "        for i , data in enumerate(train_ds):\n",
    "            embbed_drug, onehot_task, embbed_task, lbl, task_name = data\n",
    "            l_train.append(embbed_drug[0])\n",
    "            r_train.append(embbed_task)\n",
    "            lbls_train.append(lbl.tolist())\n",
    "        \n",
    "        for i , data in enumerate(valid_ds):\n",
    "            embbed_drug, onehot_task, embbed_task, lbl, task_name = data\n",
    "            l_valid.append(embbed_drug[0])\n",
    "            r_valid.append(embbed_task)\n",
    "            lbls_valid.append(lbl.tolist())\n",
    "\n",
    "        l_train = np.array(l_train).reshape(-1,128,1)\n",
    "        r_train = np.array(r_train).reshape(-1,512,1)\n",
    "        lbls_train = np.array(lbls_train)\n",
    "\n",
    "        l_valid = np.array(l_valid).reshape(-1,128,1)\n",
    "        r_valid = np.array(r_valid).reshape(-1,512,1)\n",
    "        lbls_valid = np.array(lbls_valid)\n",
    "\n",
    "        # create neural network model\n",
    "        siamese_net = siamese_model_Canonical_tox21()\n",
    "        history = History()\n",
    "        P = siamese_net.fit([l_train, r_train], lbls_train, epochs = Epoch_S, batch_size = 128, callbacks=[history])\n",
    "\n",
    "        for j in range(100):\n",
    "            C=1\n",
    "            Before = int(P.history['accuracy'][-1]*100)\n",
    "            for i in range(2,Epoch_S+1):\n",
    "                if  int(P.history['accuracy'][-i]*100) == Before:\n",
    "                    C=C+1\n",
    "                else:\n",
    "                    C=1\n",
    "                Before=int(P.history['accuracy'][-i]*100)\n",
    "                print(Before)\n",
    "            if C==Epoch_S:\n",
    "                break\n",
    "            P = siamese_net.fit([l_train, r_train], lbls_train, epochs = Epoch_S, batch_size = 128, callbacks=[history])\n",
    "        print(j+1)\n",
    "        \n",
    "        score  = siamese_net.evaluate([l_valid,r_valid], lbls_valid, verbose=1)\n",
    "        a = (score[1],score[4])\n",
    "        result.append(a)\n",
    "    \n",
    "    return result\n",
    "\n",
    "scores = evaluate_model(data_ds, 10, True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true,
    "run_control": {
     "marked": true
    }
   },
   "source": [
    "#### Dropout = 0.3 and downsampling = 0.5 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0.8456702828407288, 0.8494813442230225),\n",
       " (0.8328415751457214, 0.8326378464698792),\n",
       " (0.8669660091400146, 0.880422055721283),\n",
       " (0.8672226071357727, 0.8763396739959717),\n",
       " (0.8470814824104309, 0.8625595569610596),\n",
       " (0.8715843558311462, 0.8727011680603027),\n",
       " (0.8500128388404846, 0.8268623352050781),\n",
       " (0.8700281977653503, 0.8622535467147827),\n",
       " (0.8677187561988831, 0.8788975477218628),\n",
       " (0.8670772314071655, 0.8792339563369751)]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy= 0.8586203336715699 AUC= 0.8621389031410217 STD_AUC= 0.018696904807838016\n"
     ]
    }
   ],
   "source": [
    "acc = []\n",
    "auc = []\n",
    "for i in scores:\n",
    "    acc.append(i[0])\n",
    "    auc.append(i[1])\n",
    "\n",
    "print(f'accuracy= {np.mean(acc)} AUC= {np.mean(auc)} STD_AUC= {np.std(auc)}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "#### Dropout = 0.2 and downsampling = 0.5  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0.8568313121795654, 0.8621705770492554),\n",
       " (0.8556767106056213, 0.8610098361968994),\n",
       " (0.8727389574050903, 0.8780911564826965),\n",
       " (0.85516357421875, 0.8480740785598755),\n",
       " (0.8622193932533264, 0.8729240894317627),\n",
       " (0.8745349645614624, 0.8670076131820679),\n",
       " (0.8867077231407166, 0.8892626762390137),\n",
       " (0.8628432154655457, 0.8677256107330322),\n",
       " (0.8418014049530029, 0.8095808029174805),\n",
       " (0.847061812877655, 0.8214605450630188)]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy= 0.8615579068660736 AUC= 0.8577306985855102 STD_AUC= 0.02362893239897568\n"
     ]
    }
   ],
   "source": [
    "acc = []\n",
    "auc = []\n",
    "for i in scores:\n",
    "    acc.append(i[0])\n",
    "    auc.append(i[1])\n",
    "\n",
    "print(f'accuracy= {np.mean(acc)} AUC= {np.mean(auc)} STD_AUC= {np.std(auc)}')"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [
    "zIKQi__XAcia"
   ],
   "provenance": []
  },
  "gpuClass": "standard",
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.16"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
